{ "schema_version": 1, "generated_at": "2026-09-26T11:11:13+00:00", "generated_by": "docs/leaderboard/export_leaderboard.py", "metric": { "score": "S_macro * C_macro * Coverage_macro", "lambda": 0.5, "tau": 0.9, "reference": "sections/metrics.tex (MacroEval)", "note": "Kernel geomeans are over each mode's correct stems. The composition and sequential deltas are paired: both halves are scored over the stems that are clean on both sides, so they share a denominator." }, "environment": { "gpu": "NVIDIA B200", "gpu_count": 8, "python": "3.12.12", "torch": "2.11.0", "vllm": "0.26.0", "transformers": "5.14.1", "fastkernels": "0.1.0", "captures": "default/b200", "macroeval_universe": { "mode": "all", "models": [ "meta-llama/Llama-3.1-8B-Instruct", "Qwen/Qwen3-Next-80B-A3B-Instruct", "moonshotai/Kimi-Linear-48B-A3B-Instruct", "openai/gpt-oss-120b", "fla-hub/gla-2.7B-100B", "black-forest-labs/FLUX.1-dev", "Qwen/Qwen3-VL-235B-A22B-Instruct-FP8", "jameslahm/yolov10n", "OpenFold/OpenFold3", "BAAI/bge-m3", "Etched/oasis-500m" ], "families": [ "Dense & MoE LLMs", "Linear Attention & New Archs", "Vision / Video / Audio", "Multimodal & Encoders", "Edge & Detection", "3D / Robotics / Science", "Recommendation & Specialized", "World Models" ] }, "macroeval_config": { "tau": 0.9, "g_quantile": 0.95, "g_floor": 0.001, "g_fallback": 0.02, "g_cap": 0.5, "f": { "Dense & MoE LLMs": 0.5, "Linear Attention & New Archs": 0.5, "Vision / Video / Audio": 0.5, "Multimodal & Encoders": 0.5, "Edge & Detection": 0.5, "3D / Robotics / Science": 0.5, "Recommendation & Specialized": 0.5, "World Models": 0.5 }, "g_override": {}, "f_override": {}, "lambdas": { "blend": 0.5, "thr": 1.0, "lat": 0.0 }, "broken_threshold": 0.5, "empty_policy": "invalid", "sensitivity_scales": [ 0.5, 1.0, 2.0 ] } }, "universe": { "n_models": 11, "n_families": 8, "families": [ "Dense & MoE LLMs", "Linear Attention & New Archs", "Vision / Video / Audio", "Multimodal & Encoders", "Edge & Detection", "3D / Robotics / Science", "Recommendation & Specialized", "World Models" ], "models": [ { "id": "meta-llama/Llama-3.1-8B-Instruct", "label": "Llama-3.1-8B", "family": "Dense & MoE LLMs" }, { "id": "openai/gpt-oss-120b", "label": "GPT-OSS-120B", "family": "Dense & MoE LLMs" }, { "id": "Qwen/Qwen3-Next-80B-A3B-Instruct", "label": "Qwen3-Next-80B", "family": "Linear Attention & New Archs" }, { "id": "moonshotai/Kimi-Linear-48B-A3B-Instruct", "label": "Kimi-Linear-48B", "family": "Linear Attention & New Archs" }, { "id": "fla-hub/gla-2.7B-100B", "label": "GLA-2.7B", "family": "Linear Attention & New Archs" }, { "id": "black-forest-labs/FLUX.1-dev", "label": "FLUX.1-dev", "family": "Vision / Video / Audio" }, { "id": "Qwen/Qwen3-VL-235B-A22B-Instruct-FP8", "label": "Qwen3-VL-235B", "family": "Multimodal & Encoders" }, { "id": "jameslahm/yolov10n", "label": "YOLOv10n", "family": "Edge & Detection" }, { "id": "OpenFold/OpenFold3", "label": "OpenFold3", "family": "3D / Robotics / Science" }, { "id": "BAAI/bge-m3", "label": "BGE-M3", "family": "Recommendation & Specialized" }, { "id": "Etched/oasis-500m", "label": "Oasis-500M", "family": "World Models" } ] }, "agent_info": { "drkernel": { "name": "Dr. Kernel", "kind": "RL-trained kernel model", "models": "drkernel-14b", "loop": "8 samples x 3 generate-execute-feedback turns per operator" }, "claude": { "name": "Claude Code", "kind": "Coding agent", "models": "Claude Opus 5", "loop": "One session per operator with the default tools (5 h cap)" }, "kda": { "name": "KDA", "kind": "Long-horizon harness", "models": "Claude Opus 5 + GPT-5.6 sol reviewer", "loop": "One planning cycle, then two implement-review rounds (6 h cap)" }, "ako": { "name": "AKO", "kind": "Long-horizon harness", "models": "Claude Opus 5", "loop": "Two master-directed sub-agent rewrite rounds (5 h cap per round)" }, "codex": { "name": "Codex", "kind": "Coding agent", "models": "GPT-5.6 sol", "loop": "One session per operator, same prompt as Claude Code (5 h cap)" } }, "targets_per_level": { "1": 47, "2": 64, "3": 21 }, "excluded_stems": [ "L3:oasis_rollout" ], "boards": { "agents": { "unit": "agent", "description": "Kernel-generation agents on FastKernels L1-L3 plus the end-to-end MacroEval of their winner sets.", "rows": [ { "agent": "Dr. Kernel", "agent_key": "drkernel", "mode": "ind", "mode_label": "Independent", "open_weights": null, "kernel": { "L1": { "benched": 46, "correct": 31, "geomean": 0.933424528840685, "fast_1": 18, "fast_2": 1 }, "L2": { "benched": 52, "correct": 18, "geomean": 0.5927914128707076, "fast_1": 5, "fast_2": 0 }, "L3": { "benched": 14, "correct": 4, "geomean": 0.9746057758734402, "fast_1": 1, "fast_2": 0 } }, "composition": { "ind_to_comp_L2": { "n": 5, "from": 1.0784694490496909, "to": 1.067606868593495, "ratio": 0.9899277810180274, "broke": 0 }, "ind_to_comp_L3": { "n": 1, "from": 1.0296080071520197, "to": 1.027449079497665, "ratio": 0.9979031557259092, "broke": 0 }, "comp_to_seq_L2": { "n": 1, "from": 1.065929915984086, "to": 1.8966539735990842, "ratio": 1.7793420985356794, "broke": 0 }, "comp_to_seq_L3": { "n": 0, "from": null, "to": null, "ratio": null, "broke": 0 } }, "e2e": { "deployable": { "n_items": 11, "n_valid": 9, "coverage_macro": 0.75, "C_macro": 0.8536843204166975, "S_macro": { "blend": 0.9835959806247615, "thr": 1.0136764461275234, "lat": 0.9544081415693433 }, "score": { "blend": 0.6297603497131835, "thr": 0.6490197660260909, "lat": 0.611072449301841 }, "n_families": 8, "n_families_valid": 6, "families": { "Dense & MoE LLMs": { "n": 2, "n_valid": 2, "C_f": 1.0, "coverage_f": 1.0, "S_f": { "blend": 0.9839710588233819, "thr": 0.9858668679185979, "lat": 0.9820788953442652 } }, "Linear Attention & New Archs": { "n": 3, "n_valid": 3, "C_f": 0.985175016079499, "coverage_f": 1.0, "S_f": { "blend": 0.9517814625216422, "thr": 0.9732232182505164, "lat": 0.9308121049847915 } }, "Vision / Video / Audio": { "n": 1, "n_valid": 1, "C_f": 0.9759412256021879, "coverage_f": 1.0, "S_f": { "blend": 0.965234998995621, "thr": 1.0101303745253134, "lat": 0.922335004255165 } }, "Multimodal & Encoders": { "n": 1, "n_valid": 0, "C_f": 0.8819985902728336, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Edge & Detection": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "3D / Robotics / Science": { "n": 1, "n_valid": 1, "C_f": 0.9920662988251685, "coverage_f": 1.0, "S_f": { "blend": 1.0446559984492358, "thr": 1.055206491858485, "lat": 1.0342109942613262 } }, "Recommendation & Specialized": { "n": 1, "n_valid": 1, "C_f": 1.0, "coverage_f": 1.0, "S_f": { "blend": 1.0282195596749513, "thr": 1.027900919938134, "lat": 1.0285382981871272 } }, "World Models": { "n": 1, "n_valid": 1, "C_f": 0.9942934325538911, "coverage_f": 1.0, "S_f": { "blend": 0.9325855479923219, "thr": 1.0320464856564062, "lat": 0.8427099131789393 } } } }, "all_winners": { "n_items": 11, "n_valid": 2, "coverage_macro": 0.25, "C_macro": 0.24900828735314606, "S_macro": { "blend": 1.036405196212982, "thr": 1.0414642210397953, "lat": 1.0313707461451296 }, "score": { "blend": 0.064518370728224, "thr": 0.06483330550517444, "lat": 0.06420496578093378 }, "n_families": 8, "n_families_valid": 2, "families": { "Dense & MoE LLMs": { "n": 2, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Linear Attention & New Archs": { "n": 3, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Vision / Video / Audio": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Multimodal & Encoders": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Edge & Detection": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "3D / Robotics / Science": { "n": 1, "n_valid": 1, "C_f": 0.9920662988251685, "coverage_f": 1.0, "S_f": { "blend": 1.0446559984492358, "thr": 1.055206491858485, "lat": 1.0342109942613262 } }, "Recommendation & Specialized": { "n": 1, "n_valid": 1, "C_f": 1.0, "coverage_f": 1.0, "S_f": { "blend": 1.0282195596749513, "thr": 1.027900919938134, "lat": 1.0285382981871272 } }, "World Models": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } } } }, "drops": { "requested": 53, "kept": 31, "kept_fraction": 0.5849056603773585, "dropped_unique_kernels": 10, "dropped_by_category": { "runtime": 7, "compile": 6, "incorrect": 4 }, "status": { "ok": 10, "incomplete": 1 } }, "per_model": [ { "model": "meta-llama/Llama-3.1-8B-Instruct", "label": "Llama-3.1-8B", "family": "Dense & MoE LLMs", "outcome": "ok", "detail": null, "valid": true, "C": 1.0, "s": { "blend": 0.9763787297582486, "thr": 0.9642594081817066, "lat": 0.9886503733699498 }, "mean_d": 0.0, "n_dropped": 3, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "openai/gpt-oss-120b", "label": "GPT-OSS-120B", "family": "Dense & MoE LLMs", "outcome": "ok", "detail": null, "valid": true, "C": 1.0, "s": { "blend": 0.99162242590202, "thr": 1.0079585151182402, "lat": 0.9755510973945763 }, "mean_d": 0.0, "n_dropped": 2, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "Qwen/Qwen3-Next-80B-A3B-Instruct", "label": "Qwen3-Next-80B", "family": "Linear Attention & New Archs", "outcome": "ok", "detail": null, "valid": true, "C": 0.959383189979182, "s": { "blend": 1.002054625700267, "thr": 1.0157226124378156, "lat": 0.9885705610878835 }, "mean_d": 0.0211740382003882, "n_dropped": 1, "as_is": { "outcome": "broken", "detail": null, "valid": false } }, { "model": "moonshotai/Kimi-Linear-48B-A3B-Instruct", "label": "Kimi-Linear-48B", "family": "Linear Attention & New Archs", "outcome": "ok", "detail": null, "valid": true, "C": 0.997419935874165, "s": { "blend": 0.9877732910878174, "thr": 1.008403537656482, "lat": 0.9675651047932304 }, "mean_d": 0.00753485041963293, "n_dropped": 2, "as_is": { "outcome": "crash", "detail": "compile", "valid": false } }, { "model": "fla-hub/gla-2.7B-100B", "label": "GLA-2.7B", "family": "Linear Attention & New Archs", "outcome": "ok", "detail": null, "valid": true, "C": 0.9987219223851498, "s": { "blend": 0.8710900435472215, "thr": 0.8999697302261649, "lat": 0.8431370950403102 }, "mean_d": 0.0008717695188728506, "n_dropped": 1, "as_is": { "outcome": "broken", "detail": null, "valid": false } }, { "model": "black-forest-labs/FLUX.1-dev", "label": "FLUX.1-dev", "family": "Vision / Video / Audio", "outcome": "ok", "detail": null, "valid": true, "C": 0.9759412256021879, "s": { "blend": 0.965234998995621, "thr": 1.0101303745253134, "lat": 0.922335004255165 }, "mean_d": 0.012906470202108977, "n_dropped": 1, "as_is": { "outcome": "broken", "detail": null, "valid": false } }, { "model": "Qwen/Qwen3-VL-235B-A22B-Instruct-FP8", "label": "Qwen3-VL-235B", "family": "Multimodal & Encoders", "outcome": "ok", "detail": null, "valid": false, "C": 0.8819985902728336, "s": { "blend": 0.8901204555862069, "thr": 0.7951199625863997, "lat": 0.9964715549031405 }, "mean_d": 0.09049233135758455, "n_dropped": 3, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "jameslahm/yolov10n", "label": "YOLOv10n", "family": "Edge & Detection", "outcome": "incomplete", "detail": null, "valid": false, "C": 0.0, "s": { "blend": null, "thr": null, "lat": null }, "mean_d": null, "n_dropped": 2, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "OpenFold/OpenFold3", "label": "OpenFold3", "family": "3D / Robotics / Science", "outcome": "ok", "detail": null, "valid": true, "C": 0.9920662988251685, "s": { "blend": 1.0446559984492358, "thr": 1.055206491858485, "lat": 1.0342109942613262 }, "mean_d": 0.12070057108226195, "n_dropped": 0, "as_is": { "outcome": "ok", "detail": null, "valid": true } }, { "model": "BAAI/bge-m3", "label": "BGE-M3", "family": "Recommendation & Specialized", "outcome": "ok", "detail": null, "valid": true, "C": 1.0, "s": { "blend": 1.0282195596749513, "thr": 1.027900919938134, "lat": 1.0285382981871272 }, "mean_d": 6.641050317321949e-05, "n_dropped": 0, "as_is": { "outcome": "ok", "detail": null, "valid": true } }, { "model": "Etched/oasis-500m", "label": "Oasis-500M", "family": "World Models", "outcome": "ok", "detail": null, "valid": true, "C": 0.9942934325538911, "s": { "blend": 0.9325855479923219, "thr": 1.0320464856564062, "lat": 0.8427099131789393 }, "mean_d": 0.0033097653428741033, "n_dropped": 2, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } } ], "sensitivity": { "0.5x": { "score": 0.48266097053479146, "rank": 4, "of": 10 }, "1x": { "score": 0.6297603497131835, "rank": 4, "of": 10 }, "2x": { "score": 0.7369265420155943, "rank": 4, "of": 10 } } }, "cost": { "tokens": 33637405, "tokens_output": 10358940, "tokens_cached": 0, "n_ops": 133, "tokens_per_op": 252912.81954887218, "agent_hours": 52.57118333333333, "agent_seconds": 189256.25999999998, "median_hours_per_op": 0.26137777777777776, "source": "agent_cost.json" }, "provenance": { "run_dir": "drkernel-b200-20260919/campaign", "bench_files": 3, "bench_sha256": "9ab17075e523f90b" } }, { "agent": "Dr. Kernel", "agent_key": "drkernel", "mode": "comp", "mode_label": "Composed", "open_weights": null, "kernel": { "L1": { "benched": 18, "correct": 18, "geomean": 1.2905127885237144, "fast_1": 18, "fast_2": 1 }, "L2": { "benched": 5, "correct": 5, "geomean": 1.067606868593495, "fast_1": 4, "fast_2": 0 }, "L3": { "benched": 1, "correct": 1, "geomean": 1.027449079497665, "fast_1": 1, "fast_2": 0 } }, "composition": { "ind_to_comp_L2": { "n": 5, "from": 1.0784694490496909, "to": 1.067606868593495, "ratio": 0.9899277810180274, "broke": 0 }, "ind_to_comp_L3": { "n": 1, "from": 1.0296080071520197, "to": 1.027449079497665, "ratio": 0.9979031557259092, "broke": 0 }, "comp_to_seq_L2": { "n": 1, "from": 1.065929915984086, "to": 1.8966539735990842, "ratio": 1.7793420985356794, "broke": 0 }, "comp_to_seq_L3": { "n": 0, "from": null, "to": null, "ratio": null, "broke": 0 } }, "e2e": null, "cost": { "tokens": 33637405, "tokens_output": 10358940, "tokens_cached": 0, "n_ops": 133, "tokens_per_op": 252912.81954887218, "agent_hours": 52.57118333333333, "agent_seconds": 189256.25999999998, "median_hours_per_op": 0.26137777777777776, "source": "agent_cost.json" }, "provenance": { "run_dir": "drkernel-b200-20260919/campaign", "bench_files": 3, "bench_sha256": "9ab17075e523f90b" } }, { "agent": "Dr. Kernel", "agent_key": "drkernel", "mode": "seq", "mode_label": "Sequential", "open_weights": null, "kernel": { "L1": { "benched": 18, "correct": 18, "geomean": 1.2895565688573827, "fast_1": 18, "fast_2": 1 }, "L2": { "benched": 5, "correct": 5, "geomean": 1.4009484140090211, "fast_1": 5, "fast_2": 0 }, "L3": { "benched": 2, "correct": 2, "geomean": 1.1613804519766011, "fast_1": 2, "fast_2": 0 } }, "composition": { "ind_to_comp_L2": { "n": 5, "from": 1.0784694490496909, "to": 1.067606868593495, "ratio": 0.9899277810180274, "broke": 0 }, "ind_to_comp_L3": { "n": 1, "from": 1.0296080071520197, "to": 1.027449079497665, "ratio": 0.9979031557259092, "broke": 0 }, "comp_to_seq_L2": { "n": 1, "from": 1.065929915984086, "to": 1.8966539735990842, "ratio": 1.7793420985356794, "broke": 0 }, "comp_to_seq_L3": { "n": 0, "from": null, "to": null, "ratio": null, "broke": 0 } }, "e2e": { "deployable": { "n_items": 11, "n_valid": 9, "coverage_macro": 0.75, "C_macro": 0.7465119453042137, "S_macro": { "blend": 0.9896336074869349, "thr": 1.0224567124775163, "lat": 0.9578641962205718 }, "score": { "blend": 0.5540799820976238, "thr": 0.5724571120657065, "lat": 0.5362927983434072 }, "n_families": 8, "n_families_valid": 6, "families": { "Dense & MoE LLMs": { "n": 2, "n_valid": 2, "C_f": 1.0, "coverage_f": 1.0, "S_f": { "blend": 1.0010661506694116, "thr": 1.0138172559462397, "lat": 0.9884754201394396 } }, "Linear Attention & New Archs": { "n": 3, "n_valid": 3, "C_f": 0.9860350374547773, "coverage_f": 1.0, "S_f": { "blend": 0.9456501457580748, "thr": 0.9717692008609752, "lat": 0.9202331143855664 } }, "Vision / Video / Audio": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Multimodal & Encoders": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Edge & Detection": { "n": 1, "n_valid": 1, "C_f": 0.9997007935998725, "coverage_f": 1.0, "S_f": { "blend": 1.0210599091162889, "thr": 1.0751870525323761, "lat": 0.9696576382212068 } }, "3D / Robotics / Science": { "n": 1, "n_valid": 1, "C_f": 0.9920662988251685, "coverage_f": 1.0, "S_f": { "blend": 1.017635365129087, "thr": 1.0384248792680937, "lat": 0.9972620620292845 } }, "Recommendation & Specialized": { "n": 1, "n_valid": 1, "C_f": 1.0, "coverage_f": 1.0, "S_f": { "blend": 1.03299676633641, "thr": 1.0283935238755737, "lat": 1.0376206135955663 } }, "World Models": { "n": 1, "n_valid": 1, "C_f": 0.9942934325538911, "coverage_f": 1.0, "S_f": { "blend": 0.9245086174123242, "thr": 1.0100161763600182, "lat": 0.8462400936487429 } } } }, "all_winners": { "n_items": 11, "n_valid": 2, "coverage_macro": 0.25, "C_macro": 0.24900828735314606, "S_macro": { "blend": 1.0252872970479634, "thr": 1.0333970296408743, "lat": 1.0172412067540353 }, "score": { "blend": 0.06382625847071242, "thr": 0.06433110612667561, "lat": 0.06332537267971747 }, "n_families": 8, "n_families_valid": 2, "families": { "Dense & MoE LLMs": { "n": 2, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Linear Attention & New Archs": { "n": 3, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Vision / Video / Audio": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Multimodal & Encoders": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Edge & Detection": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "3D / Robotics / Science": { "n": 1, "n_valid": 1, "C_f": 0.9920662988251685, "coverage_f": 1.0, "S_f": { "blend": 1.017635365129087, "thr": 1.0384248792680937, "lat": 0.9972620620292845 } }, "Recommendation & Specialized": { "n": 1, "n_valid": 1, "C_f": 1.0, "coverage_f": 1.0, "S_f": { "blend": 1.03299676633641, "thr": 1.0283935238755737, "lat": 1.0376206135955663 } }, "World Models": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } } } }, "drops": { "requested": 54, "kept": 25, "kept_fraction": 0.46296296296296297, "dropped_unique_kernels": 11, "dropped_by_category": { "runtime": 7, "compile": 6, "incorrect": 5 }, "status": { "ok": 9, "incomplete": 2 } }, "per_model": [ { "model": "meta-llama/Llama-3.1-8B-Instruct", "label": "Llama-3.1-8B", "family": "Dense & MoE LLMs", "outcome": "ok", "detail": null, "valid": true, "C": 1.0, "s": { "blend": 1.0075053619469907, "thr": 1.014281339874648, "lat": 1.0007746514171165 }, "mean_d": 0.0, "n_dropped": 3, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "openai/gpt-oss-120b", "label": "GPT-OSS-120B", "family": "Dense & MoE LLMs", "outcome": "ok", "detail": null, "valid": true, "C": 1.0, "s": { "blend": 0.9946680939538262, "thr": 1.0133533843592049, "lat": 0.9763273428601255 }, "mean_d": 0.0, "n_dropped": 2, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "Qwen/Qwen3-Next-80B-A3B-Instruct", "label": "Qwen3-Next-80B", "family": "Linear Attention & New Archs", "outcome": "ok", "detail": null, "valid": true, "C": 0.959383189979182, "s": { "blend": 1.0054705188280935, "thr": 1.0227535240732244, "lat": 0.9884795705285242 }, "mean_d": 0.0211740382003882, "n_dropped": 1, "as_is": { "outcome": "broken", "detail": null, "valid": false } }, { "model": "moonshotai/Kimi-Linear-48B-A3B-Instruct", "label": "Kimi-Linear-48B", "family": "Linear Attention & New Archs", "outcome": "ok", "detail": null, "valid": true, "C": 1.0, "s": { "blend": 0.9756859732621957, "thr": 0.9897744213016222, "lat": 0.961798060176884 }, "mean_d": 0.0, "n_dropped": 2, "as_is": { "outcome": "crash", "detail": "compile", "valid": false } }, { "model": "fla-hub/gla-2.7B-100B", "label": "GLA-2.7B", "family": "Linear Attention & New Archs", "outcome": "ok", "detail": null, "valid": true, "C": 0.9987219223851498, "s": { "blend": 0.862009552858531, "thr": 0.9065299994800303, "lat": 0.8196755426136707 }, "mean_d": 0.0008717695188728506, "n_dropped": 2, "as_is": { "outcome": "broken", "detail": null, "valid": false } }, { "model": "black-forest-labs/FLUX.1-dev", "label": "FLUX.1-dev", "family": "Vision / Video / Audio", "outcome": "incomplete", "detail": null, "valid": false, "C": 0.0, "s": { "blend": null, "thr": null, "lat": null }, "mean_d": null, "n_dropped": 1, "as_is": { "outcome": "broken", "detail": null, "valid": false } }, { "model": "Qwen/Qwen3-VL-235B-A22B-Instruct-FP8", "label": "Qwen3-VL-235B", "family": "Multimodal & Encoders", "outcome": "incomplete", "detail": null, "valid": false, "C": 0.0, "s": { "blend": null, "thr": null, "lat": null }, "mean_d": null, "n_dropped": 3, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "jameslahm/yolov10n", "label": "YOLOv10n", "family": "Edge & Detection", "outcome": "ok", "detail": null, "valid": true, "C": 0.9997007935998725, "s": { "blend": 1.0210599091162889, "thr": 1.0751870525323761, "lat": 0.9696576382212068 }, "mean_d": 0.00035306747763817585, "n_dropped": 1, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "OpenFold/OpenFold3", "label": "OpenFold3", "family": "3D / Robotics / Science", "outcome": "ok", "detail": null, "valid": true, "C": 0.9920662988251685, "s": { "blend": 1.017635365129087, "thr": 1.0384248792680937, "lat": 0.9972620620292845 }, "mean_d": 0.12070057108226195, "n_dropped": 0, "as_is": { "outcome": "ok", "detail": null, "valid": true } }, { "model": "BAAI/bge-m3", "label": "BGE-M3", "family": "Recommendation & Specialized", "outcome": "ok", "detail": null, "valid": true, "C": 1.0, "s": { "blend": 1.03299676633641, "thr": 1.0283935238755737, "lat": 1.0376206135955663 }, "mean_d": 6.641050317321949e-05, "n_dropped": 0, "as_is": { "outcome": "ok", "detail": null, "valid": true } }, { "model": "Etched/oasis-500m", "label": "Oasis-500M", "family": "World Models", "outcome": "ok", "detail": null, "valid": true, "C": 0.9942934325538911, "s": { "blend": 0.9245086174123242, "thr": 1.0100161763600182, "lat": 0.8462400936487429 }, "mean_d": 0.0033097653428741033, "n_dropped": 3, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } } ], "sensitivity": { "0.5x": { "score": 0.43919243159557864, "rank": 5, "of": 10 }, "1x": { "score": 0.5540799820976238, "rank": 5, "of": 10 }, "2x": { "score": 0.5558054539944982, "rank": 6, "of": 10 } } }, "cost": { "tokens": 23287020, "tokens_output": 7958397, "tokens_cached": 0, "n_ops": 86, "tokens_per_op": 270779.3023255814, "agent_hours": 32.102808333333314, "agent_seconds": 115570.10999999993, "median_hours_per_op": 0.3449291666666666, "source": "agent_cost.json" }, "provenance": { "run_dir": "drkernel-b200-20260919/campaign", "bench_files": 3, "bench_sha256": "9ab17075e523f90b" } }, { "agent": "Claude Code", "agent_key": "claude", "mode": "ind", "mode_label": "Independent", "open_weights": false, "kernel": { "L1": { "benched": 47, "correct": 47, "geomean": 1.809879081958854, "fast_1": 42, "fast_2": 10 }, "L2": { "benched": 64, "correct": 64, "geomean": 2.660989925551296, "fast_1": 56, "fast_2": 37 }, "L3": { "benched": 21, "correct": 20, "geomean": 4.739277953998551, "fast_1": 19, "fast_2": 16 } }, "composition": { "ind_to_comp_L2": { "n": 54, "from": 3.2290003858323, "to": 3.51914367236448, "ratio": 1.0898554511808747, "broke": 2 }, "ind_to_comp_L3": { "n": 14, "from": 6.344300521708754, "to": 6.284771699116286, "ratio": 0.9906169604688848, "broke": 4 }, "comp_to_seq_L2": { "n": 50, "from": 3.677215246001829, "to": 3.625115202092557, "ratio": 0.9858316578106436, "broke": 1 }, "comp_to_seq_L3": { "n": 14, "from": 6.284771699116286, "to": 6.111550200527527, "ratio": 0.9724379011869093, "broke": 0 } }, "e2e": { "deployable": { "n_items": 11, "n_valid": 6, "coverage_macro": 0.5208333333333333, "C_macro": 0.6123815859474737, "S_macro": { "blend": 1.09216639857893, "thr": 1.07896605087575, "lat": 1.1055282427252502 }, "score": { "blend": 0.34834509962515925, "thr": 0.3441348653222782, "lat": 0.35260684301552725 }, "n_families": 8, "n_families_valid": 5, "families": { "Dense & MoE LLMs": { "n": 2, "n_valid": 1, "C_f": 0.4871338789191956, "coverage_f": 0.5, "S_f": { "blend": 1.0094461346978496, "thr": 1.0257851010302332, "lat": 0.9933674195823556 } }, "Linear Attention & New Archs": { "n": 3, "n_valid": 2, "C_f": 0.6650140994104997, "coverage_f": 0.6666666666666666, "S_f": { "blend": 1.2046928464781663, "thr": 1.1301684612223075, "lat": 1.2841314407111162 } }, "Vision / Video / Audio": { "n": 1, "n_valid": 1, "C_f": 0.9809836744340481, "coverage_f": 1.0, "S_f": { "blend": 1.2297748799305885, "thr": 1.1577860443107884, "lat": 1.3062398383015317 } }, "Multimodal & Encoders": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Edge & Detection": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "3D / Robotics / Science": { "n": 1, "n_valid": 0, "C_f": 0.8332126487970875, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Recommendation & Specialized": { "n": 1, "n_valid": 1, "C_f": 0.9998849356769218, "coverage_f": 1.0, "S_f": { "blend": 1.0685247722410947, "thr": 1.0715848325675124, "lat": 1.0654734503448198 } }, "World Models": { "n": 1, "n_valid": 1, "C_f": 0.9328234503420373, "coverage_f": 1.0, "S_f": { "blend": 0.9724658544519503, "thr": 1.0166804241999716, "lat": 0.9301741388589512 } } } }, "all_winners": { "n_items": 11, "n_valid": 2, "coverage_macro": 0.25, "C_macro": 0.2415885482523699, "S_macro": { "blend": 1.019364437108001, "thr": 1.0437716810399649, "lat": 0.9955279248477001 }, "score": { "blend": 0.06156669362525405, "thr": 0.0630408212823452, "lat": 0.060127036527162565 }, "n_families": 8, "n_families_valid": 2, "families": { "Dense & MoE LLMs": { "n": 2, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Linear Attention & New Archs": { "n": 3, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Vision / Video / Audio": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Multimodal & Encoders": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Edge & Detection": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "3D / Robotics / Science": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Recommendation & Specialized": { "n": 1, "n_valid": 1, "C_f": 0.9998849356769218, "coverage_f": 1.0, "S_f": { "blend": 1.0685247722410947, "thr": 1.0715848325675124, "lat": 1.0654734503448198 } }, "World Models": { "n": 1, "n_valid": 1, "C_f": 0.9328234503420373, "coverage_f": 1.0, "S_f": { "blend": 0.9724658544519503, "thr": 1.0166804241999716, "lat": 0.9301741388589512 } } } }, "drops": { "requested": 175, "kept": 85, "kept_fraction": 0.4857142857142857, "dropped_unique_kernels": 18, "dropped_by_category": { "compile": 11, "runtime": 11, "timeout": 1, "incorrect": 2 }, "status": { "incomplete": 4, "ok": 7 } }, "per_model": [ { "model": "meta-llama/Llama-3.1-8B-Instruct", "label": "Llama-3.1-8B", "family": "Dense & MoE LLMs", "outcome": "incomplete", "detail": null, "valid": false, "C": 0.0, "s": { "blend": null, "thr": null, "lat": null }, "mean_d": null, "n_dropped": 3, "as_is": { "outcome": "crash", "detail": "compile", "valid": false } }, { "model": "openai/gpt-oss-120b", "label": "GPT-OSS-120B", "family": "Dense & MoE LLMs", "outcome": "ok", "detail": null, "valid": true, "C": 0.9742677578383911, "s": { "blend": 1.0094461346978496, "thr": 1.0257851010302332, "lat": 0.9933674195823556 }, "mean_d": 0.013512263838642862, "n_dropped": 3, "as_is": { "outcome": "crash", "detail": "compile", "valid": false } }, { "model": "Qwen/Qwen3-Next-80B-A3B-Instruct", "label": "Qwen3-Next-80B", "family": "Linear Attention & New Archs", "outcome": "incomplete", "detail": null, "valid": false, "C": 0.0, "s": { "blend": null, "thr": null, "lat": null }, "mean_d": null, "n_dropped": 3, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "moonshotai/Kimi-Linear-48B-A3B-Instruct", "label": "Kimi-Linear-48B", "family": "Linear Attention & New Archs", "outcome": "ok", "detail": null, "valid": true, "C": 0.995753623103134, "s": { "blend": 0.9887669355005043, "thr": 1.007585619793785, "lat": 0.9702997279170669 }, "mean_d": 0.02359975028377164, "n_dropped": 5, "as_is": { "outcome": "crash", "detail": "compile", "valid": false } }, { "model": "fla-hub/gla-2.7B-100B", "label": "GLA-2.7B", "family": "Linear Attention & New Archs", "outcome": "ok", "detail": null, "valid": true, "C": 0.9992886751283654, "s": { "blend": 1.4677724368089236, "thr": 1.2676647280883289, "lat": 1.699468225723082 }, "mean_d": 0.0005112011109457398, "n_dropped": 1, "as_is": { "outcome": "broken", "detail": null, "valid": false } }, { "model": "black-forest-labs/FLUX.1-dev", "label": "FLUX.1-dev", "family": "Vision / Video / Audio", "outcome": "ok", "detail": null, "valid": true, "C": 0.9809836744340481, "s": { "blend": 1.2297748799305885, "thr": 1.1577860443107884, "lat": 1.3062398383015317 }, "mean_d": 0.01034278021254107, "n_dropped": 1, "as_is": { "outcome": "broken", "detail": null, "valid": false } }, { "model": "Qwen/Qwen3-VL-235B-A22B-Instruct-FP8", "label": "Qwen3-VL-235B", "family": "Multimodal & Encoders", "outcome": "incomplete", "detail": null, "valid": false, "C": 0.0, "s": { "blend": null, "thr": null, "lat": null }, "mean_d": null, "n_dropped": 3, "as_is": { "outcome": "crash", "detail": "compile", "valid": false } }, { "model": "jameslahm/yolov10n", "label": "YOLOv10n", "family": "Edge & Detection", "outcome": "incomplete", "detail": null, "valid": false, "C": 0.0, "s": { "blend": null, "thr": null, "lat": null }, "mean_d": null, "n_dropped": 0, "as_is": { "outcome": "broken", "detail": null, "valid": false } }, { "model": "OpenFold/OpenFold3", "label": "OpenFold3", "family": "3D / Robotics / Science", "outcome": "ok", "detail": null, "valid": false, "C": 0.8332126487970875, "s": { "blend": 1.4254221861794807, "thr": 1.4042073588871946, "lat": 1.4469575280269666 }, "mean_d": 0.20696169494970407, "n_dropped": 6, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "BAAI/bge-m3", "label": "BGE-M3", "family": "Recommendation & Specialized", "outcome": "ok", "detail": null, "valid": true, "C": 0.9998849356769218, "s": { "blend": 1.0685247722410947, "thr": 1.0715848325675124, "lat": 1.0654734503448198 }, "mean_d": 0.00025077070667197526, "n_dropped": 0, "as_is": { "outcome": "ok", "detail": null, "valid": true } }, { "model": "Etched/oasis-500m", "label": "Oasis-500M", "family": "World Models", "outcome": "ok", "detail": null, "valid": true, "C": 0.9328234503420373, "s": { "blend": 0.9724658544519503, "thr": 1.0166804241999716, "lat": 0.9301741388589512 }, "mean_d": 0.03452109827932349, "n_dropped": 0, "as_is": { "outcome": "ok", "detail": null, "valid": true } } ], "sensitivity": { "0.5x": { "score": 0.23977805684482997, "rank": 8, "of": 10 }, "1x": { "score": 0.34834509962515925, "rank": 8, "of": 10 }, "2x": { "score": 0.4707386362857802, "rank": 7, "of": 10 } } }, "cost": { "tokens": 4354090521, "tokens_output": 31380043, "tokens_cached": 4242086474, "n_ops": 133, "tokens_per_op": 32737522.714285713, "agent_hours": 1170.7130055555563, "agent_seconds": 4214566.820000002, "median_hours_per_op": 7.271027777777778, "source": "agent_cost.json" }, "provenance": { "run_dir": "claude-fk-runs", "bench_files": 3, "bench_sha256": "76492e9181a6d8eb" } }, { "agent": "Claude Code", "agent_key": "claude", "mode": "comp", "mode_label": "Composed", "open_weights": false, "kernel": { "L1": { "benched": 43, "correct": 42, "geomean": 1.7912326917166819, "fast_1": 40, "fast_2": 10 }, "L2": { "benched": 56, "correct": 54, "geomean": 3.51914367236448, "fast_1": 54, "fast_2": 36 }, "L3": { "benched": 18, "correct": 14, "geomean": 6.284771699116286, "fast_1": 14, "fast_2": 12 } }, "composition": { "ind_to_comp_L2": { "n": 54, "from": 3.2290003858323, "to": 3.51914367236448, "ratio": 1.0898554511808747, "broke": 2 }, "ind_to_comp_L3": { "n": 14, "from": 6.344300521708754, "to": 6.284771699116286, "ratio": 0.9906169604688848, "broke": 4 }, "comp_to_seq_L2": { "n": 50, "from": 3.677215246001829, "to": 3.625115202092557, "ratio": 0.9858316578106436, "broke": 1 }, "comp_to_seq_L3": { "n": 14, "from": 6.284771699116286, "to": 6.111550200527527, "ratio": 0.9724379011869093, "broke": 0 } }, "e2e": null, "cost": { "tokens": 4354090521, "tokens_output": 31380043, "tokens_cached": 4242086474, "n_ops": 133, "tokens_per_op": 32737522.714285713, "agent_hours": 1170.7130055555563, "agent_seconds": 4214566.820000002, "median_hours_per_op": 7.271027777777778, "source": "agent_cost.json" }, "provenance": { "run_dir": "claude-fk-runs", "bench_files": 3, "bench_sha256": "76492e9181a6d8eb" } }, { "agent": "Claude Code", "agent_key": "claude", "mode": "seq", "mode_label": "Sequential", "open_weights": false, "kernel": { "L1": { "benched": 43, "correct": 42, "geomean": 1.7568058781266032, "fast_1": 40, "fast_2": 10 }, "L2": { "benched": 58, "correct": 57, "geomean": 3.2091076503653637, "fast_1": 57, "fast_2": 37 }, "L3": { "benched": 20, "correct": 20, "geomean": 4.4993133975955795, "fast_1": 20, "fast_2": 15 } }, "composition": { "ind_to_comp_L2": { "n": 54, "from": 3.2290003858323, "to": 3.51914367236448, "ratio": 1.0898554511808747, "broke": 2 }, "ind_to_comp_L3": { "n": 14, "from": 6.344300521708754, "to": 6.284771699116286, "ratio": 0.9906169604688848, "broke": 4 }, "comp_to_seq_L2": { "n": 50, "from": 3.677215246001829, "to": 3.625115202092557, "ratio": 0.9858316578106436, "broke": 1 }, "comp_to_seq_L3": { "n": 14, "from": 6.284771699116286, "to": 6.111550200527527, "ratio": 0.9724379011869093, "broke": 0 } }, "e2e": { "deployable": { "n_items": 11, "n_valid": 7, "coverage_macro": 0.5625, "C_macro": 0.547251092073871, "S_macro": { "blend": 1.113247062359129, "thr": 1.089885914861282, "lat": 1.137108944112713 }, "score": { "blend": 0.342689439726035, "thr": 0.3354982071433687, "lat": 0.3500348127033648 }, "n_families": 8, "n_families_valid": 5, "families": { "Dense & MoE LLMs": { "n": 2, "n_valid": 1, "C_f": 0.4801590284423802, "coverage_f": 0.5, "S_f": { "blend": 1.0141353041957222, "thr": 1.0154791664500045, "lat": 1.0127932203784755 } }, "Linear Attention & New Archs": { "n": 3, "n_valid": 3, "C_f": 0.9859449677199946, "coverage_f": 1.0, "S_f": { "blend": 1.159861111295113, "thr": 1.0968405648252362, "lat": 1.226502593573463 } }, "Vision / Video / Audio": { "n": 1, "n_valid": 1, "C_f": 0.9787172690880728, "coverage_f": 1.0, "S_f": { "blend": 1.3041789522297722, "thr": 1.158053667450203, "lat": 1.468742587020291 } }, "Multimodal & Encoders": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Edge & Detection": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "3D / Robotics / Science": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Recommendation & Specialized": { "n": 1, "n_valid": 1, "C_f": 0.9999726502085807, "coverage_f": 1.0, "S_f": { "blend": 1.0682393157227215, "thr": 1.0511234315736235, "lat": 1.0856339049995005 } }, "World Models": { "n": 1, "n_valid": 1, "C_f": 0.9332148211319401, "coverage_f": 1.0, "S_f": { "blend": 1.0433983729252085, "thr": 1.1342486596788703, "lat": 0.9598249513745959 } } } }, "all_winners": { "n_items": 11, "n_valid": 1, "coverage_macro": 0.125, "C_macro": 0.12499658127607259, "S_macro": { "blend": 1.0682393157227215, "thr": 1.0511234315736235, "lat": 1.0856339049995005 }, "score": { "blend": 0.016690782806253916, "thr": 0.016423354430734595, "lat": 0.01696256583029127 }, "n_families": 8, "n_families_valid": 1, "families": { "Dense & MoE LLMs": { "n": 2, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Linear Attention & New Archs": { "n": 3, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Vision / Video / Audio": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Multimodal & Encoders": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Edge & Detection": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "3D / Robotics / Science": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Recommendation & Specialized": { "n": 1, "n_valid": 1, "C_f": 0.9999726502085807, "coverage_f": 1.0, "S_f": { "blend": 1.0682393157227215, "thr": 1.0511234315736235, "lat": 1.0856339049995005 } }, "World Models": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } } } }, "drops": { "requested": 190, "kept": 93, "kept_fraction": 0.48947368421052634, "dropped_unique_kernels": 18, "dropped_by_category": { "compile": 12, "runtime": 8, "incorrect": 3, "timeout": 1 }, "status": { "incomplete": 4, "ok": 7 } }, "per_model": [ { "model": "meta-llama/Llama-3.1-8B-Instruct", "label": "Llama-3.1-8B", "family": "Dense & MoE LLMs", "outcome": "incomplete", "detail": null, "valid": false, "C": 0.0, "s": { "blend": null, "thr": null, "lat": null }, "mean_d": null, "n_dropped": 4, "as_is": { "outcome": "crash", "detail": "compile", "valid": false } }, { "model": "openai/gpt-oss-120b", "label": "GPT-OSS-120B", "family": "Dense & MoE LLMs", "outcome": "ok", "detail": null, "valid": true, "C": 0.9603180568847604, "s": { "blend": 1.0141353041957222, "thr": 1.0154791664500045, "lat": 1.0127932203784755 }, "mean_d": 0.020676289614504538, "n_dropped": 3, "as_is": { "outcome": "crash", "detail": "compile", "valid": false } }, { "model": "Qwen/Qwen3-Next-80B-A3B-Instruct", "label": "Qwen3-Next-80B", "family": "Linear Attention & New Archs", "outcome": "ok", "detail": null, "valid": true, "C": 0.9637887006596673, "s": { "blend": 1.0279440577981458, "thr": 1.022082180145543, "lat": 1.033839554674605 }, "mean_d": 0.018928813370826074, "n_dropped": 1, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "moonshotai/Kimi-Linear-48B-A3B-Instruct", "label": "Kimi-Linear-48B", "family": "Linear Attention & New Archs", "outcome": "ok", "detail": null, "valid": true, "C": 0.995753623103134, "s": { "blend": 1.016421784758217, "thr": 1.01441230930341, "lat": 1.0184352408346773 }, "mean_d": 0.02359975028377164, "n_dropped": 5, "as_is": { "outcome": "crash", "detail": "compile", "valid": false } }, { "model": "fla-hub/gla-2.7B-100B", "label": "GLA-2.7B", "family": "Linear Attention & New Archs", "outcome": "ok", "detail": null, "valid": true, "C": 0.9982925793971825, "s": { "blend": 1.493394394905049, "thr": 1.2727121327127338, "lat": 1.7523419172410815 }, "mean_d": 0.0010700204589310092, "n_dropped": 2, "as_is": { "outcome": "broken", "detail": null, "valid": false } }, { "model": "black-forest-labs/FLUX.1-dev", "label": "FLUX.1-dev", "family": "Vision / Video / Audio", "outcome": "ok", "detail": null, "valid": true, "C": 0.9787172690880728, "s": { "blend": 1.3041789522297722, "thr": 1.158053667450203, "lat": 1.468742587020291 }, "mean_d": 0.011528020091099628, "n_dropped": 1, "as_is": { "outcome": "broken", "detail": null, "valid": false } }, { "model": "Qwen/Qwen3-VL-235B-A22B-Instruct-FP8", "label": "Qwen3-VL-235B", "family": "Multimodal & Encoders", "outcome": "incomplete", "detail": null, "valid": false, "C": 0.0, "s": { "blend": null, "thr": null, "lat": null }, "mean_d": null, "n_dropped": 4, "as_is": { "outcome": "crash", "detail": "compile", "valid": false } }, { "model": "jameslahm/yolov10n", "label": "YOLOv10n", "family": "Edge & Detection", "outcome": "incomplete", "detail": null, "valid": false, "C": 0.0, "s": { "blend": null, "thr": null, "lat": null }, "mean_d": null, "n_dropped": 2, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "OpenFold/OpenFold3", "label": "OpenFold3", "family": "3D / Robotics / Science", "outcome": "incomplete", "detail": null, "valid": false, "C": 0.0, "s": { "blend": null, "thr": null, "lat": null }, "mean_d": null, "n_dropped": 1, "as_is": { "outcome": "crash", "detail": "timeout", "valid": false } }, { "model": "BAAI/bge-m3", "label": "BGE-M3", "family": "Recommendation & Specialized", "outcome": "ok", "detail": null, "valid": true, "C": 0.9999726502085807, "s": { "blend": 1.0682393157227215, "thr": 1.0511234315736235, "lat": 1.0856339049995005 }, "mean_d": 0.00013689674059257258, "n_dropped": 0, "as_is": { "outcome": "ok", "detail": null, "valid": true } }, { "model": "Etched/oasis-500m", "label": "Oasis-500M", "family": "World Models", "outcome": "ok", "detail": null, "valid": true, "C": 0.9332148211319401, "s": { "blend": 1.0433983729252085, "thr": 1.1342486596788703, "lat": 0.9598249513745959 }, "mean_d": 0.03432580425516187, "n_dropped": 1, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } } ], "sensitivity": { "0.5x": { "score": 0.26242664050513076, "rank": 7, "of": 10 }, "1x": { "score": 0.342689439726035, "rank": 9, "of": 10 }, "2x": { "score": 0.3477179570353601, "rank": 9, "of": 10 } } }, "cost": { "tokens": 3577444022, "tokens_output": 22451963, "tokens_cached": 3481597797, "n_ops": 86, "tokens_per_op": 41598186.302325584, "agent_hours": 592.0648611111112, "agent_seconds": 2131433.5000000005, "median_hours_per_op": 6.0586166666666665, "source": "agent_cost.json" }, "provenance": { "run_dir": "claude-fk-runs", "bench_files": 3, "bench_sha256": "76492e9181a6d8eb" } }, { "agent": "KDA", "agent_key": "kda", "mode": "ind", "mode_label": "Independent", "open_weights": null, "kernel": { "L1": { "benched": 47, "correct": 47, "geomean": 1.6411511513224948, "fast_1": 42, "fast_2": 8 }, "L2": { "benched": 63, "correct": 63, "geomean": 2.5595353221847708, "fast_1": 61, "fast_2": 37 }, "L3": { "benched": 20, "correct": 20, "geomean": 3.0491462136620044, "fast_1": 18, "fast_2": 14 } }, "composition": { "ind_to_comp_L2": { "n": 51, "from": 2.6658934948919626, "to": 2.6206257620868754, "ratio": 0.9830196769331471, "broke": 10 }, "ind_to_comp_L3": { "n": 11, "from": 3.905627160838026, "to": 3.4513973574739034, "ratio": 0.8836986264539755, "broke": 6 }, "comp_to_seq_L2": { "n": 47, "from": 2.669311156147895, "to": 2.3997664061003423, "ratio": 0.8990208580866477, "broke": 1 }, "comp_to_seq_L3": { "n": 11, "from": 3.4513973574739034, "to": 3.7016163454008653, "ratio": 1.0724978789779507, "broke": 0 } }, "e2e": { "deployable": { "n_items": 11, "n_valid": 11, "coverage_macro": 1.0, "C_macro": 0.9825416498789501, "S_macro": { "blend": 1.1338144837553161, "thr": 1.0759467302600714, "lat": 1.194794544579918 }, "score": { "blend": 1.1140199535255983, "thr": 1.0571624755315923, "lat": 1.1739354030979214 }, "n_families": 8, "n_families_valid": 8, "families": { "Dense & MoE LLMs": { "n": 2, "n_valid": 2, "C_f": 0.9866184333421674, "coverage_f": 1.0, "S_f": { "blend": 1.010437978385275, "thr": 1.020062212982121, "lat": 1.0009045479476226 } }, "Linear Attention & New Archs": { "n": 3, "n_valid": 3, "C_f": 0.9866792790780036, "coverage_f": 1.0, "S_f": { "blend": 1.0541595925866842, "thr": 1.045657709222543, "lat": 1.062730601841736 } }, "Vision / Video / Audio": { "n": 1, "n_valid": 1, "C_f": 0.9739321505763959, "coverage_f": 1.0, "S_f": { "blend": 1.2377591229551186, "thr": 1.2249524795321687, "lat": 1.2506996573807834 } }, "Multimodal & Encoders": { "n": 1, "n_valid": 1, "C_f": 0.9990558649346815, "coverage_f": 1.0, "S_f": { "blend": 1.1006816801697263, "thr": 1.1263073022674939, "lat": 1.0756390894583092 } }, "Edge & Detection": { "n": 1, "n_valid": 1, "C_f": 0.9955177410124512, "coverage_f": 1.0, "S_f": { "blend": 1.1327555515568741, "thr": 1.0202054915749112, "lat": 1.2577222433905129 } }, "3D / Robotics / Science": { "n": 1, "n_valid": 1, "C_f": 0.9965700416874375, "coverage_f": 1.0, "S_f": { "blend": 1.3324952167833164, "thr": 1.0686633952349658, "lat": 1.6614618884368453 } }, "Recommendation & Specialized": { "n": 1, "n_valid": 1, "C_f": 0.999854279654307, "coverage_f": 1.0, "S_f": { "blend": 1.1060487641414818, "thr": 1.1090548507918716, "lat": 1.103050825471278 } }, "World Models": { "n": 1, "n_valid": 1, "C_f": 0.9221054087461572, "coverage_f": 1.0, "S_f": { "blend": 1.127316778436545, "thr": 1.0093741999589512, "lat": 1.2590406204123628 } } } }, "all_winners": { "n_items": 11, "n_valid": 4, "coverage_macro": 0.41666666666666663, "C_macro": 0.4129140252726502, "S_macro": { "blend": 1.2082217020203403, "thr": 1.1261326514548633, "lat": 1.2962946055662237 }, "score": { "blend": 0.20787153600124633, "thr": 0.19374831920966248, "lat": 0.2230242598014882 }, "n_families": 8, "n_families_valid": 4, "families": { "Dense & MoE LLMs": { "n": 2, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Linear Attention & New Archs": { "n": 3, "n_valid": 1, "C_f": 0.33295573026306163, "coverage_f": 0.3333333333333333, "S_f": { "blend": 1.1681818636302568, "thr": 1.1077582733935238, "lat": 1.2319013085175823 } }, "Vision / Video / Audio": { "n": 1, "n_valid": 1, "C_f": 0.9739321505763959, "coverage_f": 1.0, "S_f": { "blend": 1.2377591229551186, "thr": 1.2249524795321687, "lat": 1.2506996573807834 } }, "Multimodal & Encoders": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Edge & Detection": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "3D / Robotics / Science": { "n": 1, "n_valid": 1, "C_f": 0.9965700416874375, "coverage_f": 1.0, "S_f": { "blend": 1.3324952167833164, "thr": 1.0686633952349658, "lat": 1.6614618884368453 } }, "Recommendation & Specialized": { "n": 1, "n_valid": 1, "C_f": 0.999854279654307, "coverage_f": 1.0, "S_f": { "blend": 1.1060487641414818, "thr": 1.1090548507918716, "lat": 1.103050825471278 } }, "World Models": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } } } }, "drops": { "requested": 171, "kept": 160, "kept_fraction": 0.935672514619883, "dropped_unique_kernels": 9, "dropped_by_category": { "runtime": 9, "compile": 2 }, "status": { "ok": 11 } }, "per_model": [ { "model": "meta-llama/Llama-3.1-8B-Instruct", "label": "Llama-3.1-8B", "family": "Dense & MoE LLMs", "outcome": "ok", "detail": null, "valid": true, "C": 1.0, "s": { "blend": 1.0168910639168447, "thr": 1.016860361685486, "lat": 1.0169217670752009 }, "mean_d": 0.0015254084154700445, "n_dropped": 2, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "openai/gpt-oss-120b", "label": "GPT-OSS-120B", "family": "Dense & MoE LLMs", "outcome": "ok", "detail": null, "valid": true, "C": 0.9732368666843347, "s": { "blend": 1.0040258434671538, "thr": 1.0232741461465442, "lat": 0.9851396110672016 }, "mean_d": 0.014026678524516929, "n_dropped": 3, "as_is": { "outcome": "crash", "detail": "compile", "valid": false } }, { "model": "Qwen/Qwen3-Next-80B-A3B-Instruct", "label": "Qwen3-Next-80B", "family": "Linear Attention & New Archs", "outcome": "ok", "detail": null, "valid": true, "C": 0.9627723202804732, "s": { "blend": 1.0108896638440803, "thr": 1.0189644001118061, "lat": 1.0028789154505002 }, "mean_d": 0.01948286218004404, "n_dropped": 1, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "moonshotai/Kimi-Linear-48B-A3B-Instruct", "label": "Kimi-Linear-48B", "family": "Linear Attention & New Archs", "outcome": "ok", "detail": null, "valid": true, "C": 0.998398326164353, "s": { "blend": 0.9919844853901856, "thr": 1.0128954437837694, "lat": 0.9715052281989537 }, "mean_d": 0.0178225438271818, "n_dropped": 1, "as_is": { "outcome": "crash", "detail": "compile", "valid": false } }, { "model": "fla-hub/gla-2.7B-100B", "label": "GLA-2.7B", "family": "Linear Attention & New Archs", "outcome": "ok", "detail": null, "valid": true, "C": 0.9988671907891848, "s": { "blend": 1.1681818636302568, "thr": 1.1077582733935238, "lat": 1.2319013085175823 }, "mean_d": 0.0007989143743217901, "n_dropped": 0, "as_is": { "outcome": "ok", "detail": null, "valid": true } }, { "model": "black-forest-labs/FLUX.1-dev", "label": "FLUX.1-dev", "family": "Vision / Video / Audio", "outcome": "ok", "detail": null, "valid": true, "C": 0.9739321505763959, "s": { "blend": 1.2377591229551186, "thr": 1.2249524795321687, "lat": 1.2506996573807834 }, "mean_d": 0.013890538437119031, "n_dropped": 0, "as_is": { "outcome": "ok", "detail": null, "valid": true } }, { "model": "Qwen/Qwen3-VL-235B-A22B-Instruct-FP8", "label": "Qwen3-VL-235B", "family": "Multimodal & Encoders", "outcome": "ok", "detail": null, "valid": true, "C": 0.9990558649346815, "s": { "blend": 1.1006816801697263, "thr": 1.1263073022674939, "lat": 1.0756390894583092 }, "mean_d": 0.024406002827459944, "n_dropped": 1, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "jameslahm/yolov10n", "label": "YOLOv10n", "family": "Edge & Detection", "outcome": "ok", "detail": null, "valid": true, "C": 0.9955177410124512, "s": { "blend": 1.1327555515568741, "thr": 1.0202054915749112, "lat": 1.2577222433905129 }, "mean_d": 0.0028613463394102138, "n_dropped": 2, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "OpenFold/OpenFold3", "label": "OpenFold3", "family": "3D / Robotics / Science", "outcome": "ok", "detail": null, "valid": true, "C": 0.9965700416874375, "s": { "blend": 1.3324952167833164, "thr": 1.0686633952349658, "lat": 1.6614618884368453 }, "mean_d": 0.11185546725170337, "n_dropped": 0, "as_is": { "outcome": "ok", "detail": null, "valid": true } }, { "model": "BAAI/bge-m3", "label": "BGE-M3", "family": "Recommendation & Specialized", "outcome": "ok", "detail": null, "valid": true, "C": 0.999854279654307, "s": { "blend": 1.1060487641414818, "thr": 1.1090548507918716, "lat": 1.103050825471278 }, "mean_d": 0.00026168268308457594, "n_dropped": 0, "as_is": { "outcome": "ok", "detail": null, "valid": true } }, { "model": "Etched/oasis-500m", "label": "Oasis-500M", "family": "World Models", "outcome": "ok", "detail": null, "valid": true, "C": 0.9221054087461572, "s": { "blend": 1.127316778436545, "thr": 1.0093741999589512, "lat": 1.2590406204123628 }, "mean_d": 0.03986940103566769, "n_dropped": 1, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } } ], "sensitivity": { "0.5x": { "score": 0.7764810458803155, "rank": 1, "of": 10 }, "1x": { "score": 1.1140199535255983, "rank": 1, "of": 10 }, "2x": { "score": 1.1246470388557, "rank": 1, "of": 10 } } }, "cost": { "tokens": 12309628108, "tokens_output": 62821722, "tokens_cached": 11875237912, "n_ops": 133, "tokens_per_op": 92553594.79699248, "agent_hours": 1750.6067222222218, "agent_seconds": 6302184.199999998, "median_hours_per_op": 12.327227777777777, "source": "agent_cost.json" }, "provenance": { "run_dir": "kda-fk-runs", "bench_files": 3, "bench_sha256": "a59e900f08782b78" } }, { "agent": "KDA", "agent_key": "kda", "mode": "comp", "mode_label": "Composed", "open_weights": null, "kernel": { "L1": { "benched": 43, "correct": 43, "geomean": 1.6902845459191027, "fast_1": 40, "fast_2": 9 }, "L2": { "benched": 61, "correct": 51, "geomean": 2.6206257620868754, "fast_1": 51, "fast_2": 31 }, "L3": { "benched": 17, "correct": 11, "geomean": 3.4513973574739034, "fast_1": 11, "fast_2": 9 } }, "composition": { "ind_to_comp_L2": { "n": 51, "from": 2.6658934948919626, "to": 2.6206257620868754, "ratio": 0.9830196769331471, "broke": 10 }, "ind_to_comp_L3": { "n": 11, "from": 3.905627160838026, "to": 3.4513973574739034, "ratio": 0.8836986264539755, "broke": 6 }, "comp_to_seq_L2": { "n": 47, "from": 2.669311156147895, "to": 2.3997664061003423, "ratio": 0.8990208580866477, "broke": 1 }, "comp_to_seq_L3": { "n": 11, "from": 3.4513973574739034, "to": 3.7016163454008653, "ratio": 1.0724978789779507, "broke": 0 } }, "e2e": null, "cost": { "tokens": 12309628108, "tokens_output": 62821722, "tokens_cached": 11875237912, "n_ops": 133, "tokens_per_op": 92553594.79699248, "agent_hours": 1750.6067222222218, "agent_seconds": 6302184.199999998, "median_hours_per_op": 12.327227777777777, "source": "agent_cost.json" }, "provenance": { "run_dir": "kda-fk-runs", "bench_files": 3, "bench_sha256": "a59e900f08782b78" } }, { "agent": "KDA", "agent_key": "kda", "mode": "seq", "mode_label": "Sequential", "open_weights": null, "kernel": { "L1": { "benched": 43, "correct": 43, "geomean": 1.6791690288557612, "fast_1": 40, "fast_2": 8 }, "L2": { "benched": 58, "correct": 57, "geomean": 2.301439653163445, "fast_1": 57, "fast_2": 30 }, "L3": { "benched": 20, "correct": 20, "geomean": 2.864666367239763, "fast_1": 20, "fast_2": 14 } }, "composition": { "ind_to_comp_L2": { "n": 51, "from": 2.6658934948919626, "to": 2.6206257620868754, "ratio": 0.9830196769331471, "broke": 10 }, "ind_to_comp_L3": { "n": 11, "from": 3.905627160838026, "to": 3.4513973574739034, "ratio": 0.8836986264539755, "broke": 6 }, "comp_to_seq_L2": { "n": 47, "from": 2.669311156147895, "to": 2.3997664061003423, "ratio": 0.8990208580866477, "broke": 1 }, "comp_to_seq_L3": { "n": 11, "from": 3.4513973574739034, "to": 3.7016163454008653, "ratio": 1.0724978789779507, "broke": 0 } }, "e2e": { "deployable": { "n_items": 11, "n_valid": 11, "coverage_macro": 1.0, "C_macro": 0.982814563366831, "S_macro": { "blend": 1.044174085843535, "thr": 1.0334642716118427, "lat": 1.0549948861286673 }, "score": { "blend": 1.026229498257274, "thr": 1.0157037368594133, "lat": 1.0368643383647858 }, "n_families": 8, "n_families_valid": 8, "families": { "Dense & MoE LLMs": { "n": 2, "n_valid": 2, "C_f": 0.9866184333421674, "coverage_f": 1.0, "S_f": { "blend": 1.013914512892381, "thr": 1.006660835910658, "lat": 1.0212204575573975 } }, "Linear Attention & New Archs": { "n": 3, "n_valid": 3, "C_f": 0.9881339159572143, "coverage_f": 1.0, "S_f": { "blend": 1.0540974703326653, "thr": 1.058701403166052, "lat": 1.0495135584395276 } }, "Vision / Video / Audio": { "n": 1, "n_valid": 1, "C_f": 0.9807135259081109, "coverage_f": 1.0, "S_f": { "blend": 1.2523671513808636, "thr": 1.2012769228510898, "lat": 1.3056302439701828 } }, "Multimodal & Encoders": { "n": 1, "n_valid": 1, "C_f": 0.9990899221600671, "coverage_f": 1.0, "S_f": { "blend": 1.1413509933751536, "thr": 1.1380032231909216, "lat": 1.144708612007068 } }, "Edge & Detection": { "n": 1, "n_valid": 1, "C_f": 0.9973237955792069, "coverage_f": 1.0, "S_f": { "blend": 1.0136153509660921, "thr": 1.1890509877048334, "lat": 0.8640639386686728 } }, "3D / Robotics / Science": { "n": 1, "n_valid": 1, "C_f": 0.9874953195835267, "coverage_f": 1.0, "S_f": { "blend": 0.7209618183429709, "thr": 0.6158064010602652, "lat": 0.8440736286817758 } }, "Recommendation & Specialized": { "n": 1, "n_valid": 1, "C_f": 0.9999992253051744, "coverage_f": 1.0, "S_f": { "blend": 1.0724222808846824, "thr": 1.071721979079743, "lat": 1.073123040292085 } }, "World Models": { "n": 1, "n_valid": 1, "C_f": 0.9231423690991811, "coverage_f": 1.0, "S_f": { "blend": 1.1803179543573088, "thr": 1.1381346204919784, "lat": 1.224064753232802 } } } }, "all_winners": { "n_items": 11, "n_valid": 6, "coverage_macro": 0.5833333333333333, "C_macro": 0.5683356142248028, "S_macro": { "blend": 1.0416151948788754, "thr": 0.9932994472503714, "lat": 1.0922811013395053 }, "score": { "blend": 0.3453257567476344, "thr": 0.32930768001961486, "lat": 0.36212297953762684 }, "n_families": 8, "n_families_valid": 5, "families": { "Dense & MoE LLMs": { "n": 2, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Linear Attention & New Archs": { "n": 3, "n_valid": 2, "C_f": 0.6553344739024299, "coverage_f": 0.6666666666666666, "S_f": { "blend": 1.0728222159043612, "thr": 1.0716127306921566, "lat": 1.0740330662127768 } }, "Vision / Video / Audio": { "n": 1, "n_valid": 1, "C_f": 0.9807135259081109, "coverage_f": 1.0, "S_f": { "blend": 1.2523671513808636, "thr": 1.2012769228510898, "lat": 1.3056302439701828 } }, "Multimodal & Encoders": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Edge & Detection": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "3D / Robotics / Science": { "n": 1, "n_valid": 1, "C_f": 0.9874953195835267, "coverage_f": 1.0, "S_f": { "blend": 0.7209618183429709, "thr": 0.6158064010602652, "lat": 0.8440736286817758 } }, "Recommendation & Specialized": { "n": 1, "n_valid": 1, "C_f": 0.9999992253051744, "coverage_f": 1.0, "S_f": { "blend": 1.0724222808846824, "thr": 1.071721979079743, "lat": 1.073123040292085 } }, "World Models": { "n": 1, "n_valid": 1, "C_f": 0.9231423690991811, "coverage_f": 1.0, "S_f": { "blend": 1.1803179543573088, "thr": 1.1381346204919784, "lat": 1.224064753232802 } } } }, "drops": { "requested": 193, "kept": 178, "kept_fraction": 0.9222797927461139, "dropped_unique_kernels": 11, "dropped_by_category": { "compile": 9, "import": 1, "runtime": 5 }, "status": { "ok": 11 } }, "per_model": [ { "model": "meta-llama/Llama-3.1-8B-Instruct", "label": "Llama-3.1-8B", "family": "Dense & MoE LLMs", "outcome": "ok", "detail": null, "valid": true, "C": 1.0, "s": { "blend": 1.0103223282644156, "thr": 1.0077651109060155, "lat": 1.0128860345958386 }, "mean_d": 0.0, "n_dropped": 6, "as_is": { "outcome": "crash", "detail": "compile", "valid": false } }, { "model": "openai/gpt-oss-120b", "label": "GPT-OSS-120B", "family": "Dense & MoE LLMs", "outcome": "ok", "detail": null, "valid": true, "C": 0.9732368666843347, "s": { "blend": 1.0175194694744452, "thr": 1.0055577709425698, "lat": 1.029623459415031 }, "mean_d": 0.014026678524516929, "n_dropped": 3, "as_is": { "outcome": "crash", "detail": "compile", "valid": false } }, { "model": "Qwen/Qwen3-Next-80B-A3B-Instruct", "label": "Qwen3-Next-80B", "family": "Linear Attention & New Archs", "outcome": "ok", "detail": null, "valid": true, "C": 0.9667727898881495, "s": { "blend": 1.0151034108909032, "thr": 1.0300286087177721, "lat": 1.000394480387374 }, "mean_d": 0.017486627845813354, "n_dropped": 0, "as_is": { "outcome": "ok", "detail": null, "valid": true } }, { "model": "moonshotai/Kimi-Linear-48B-A3B-Instruct", "label": "Kimi-Linear-48B", "family": "Linear Attention & New Archs", "outcome": "ok", "detail": null, "valid": true, "C": 0.998398326164353, "s": { "blend": 1.017622724787568, "thr": 1.0333435602678482, "lat": 1.0021410591997622 }, "mean_d": 0.0178225438271818, "n_dropped": 3, "as_is": { "outcome": "crash", "detail": "compile", "valid": false } }, { "model": "fla-hub/gla-2.7B-100B", "label": "GLA-2.7B", "family": "Linear Attention & New Archs", "outcome": "ok", "detail": null, "valid": true, "C": 0.9992306318191403, "s": { "blend": 1.1338229135963767, "thr": 1.1148756790464538, "lat": 1.1530921550784057 }, "mean_d": 0.0006015660894365237, "n_dropped": 0, "as_is": { "outcome": "ok", "detail": null, "valid": true } }, { "model": "black-forest-labs/FLUX.1-dev", "label": "FLUX.1-dev", "family": "Vision / Video / Audio", "outcome": "ok", "detail": null, "valid": true, "C": 0.9807135259081109, "s": { "blend": 1.2523671513808636, "thr": 1.2012769228510898, "lat": 1.3056302439701828 }, "mean_d": 0.010553115665076817, "n_dropped": 0, "as_is": { "outcome": "ok", "detail": null, "valid": true } }, { "model": "Qwen/Qwen3-VL-235B-A22B-Instruct-FP8", "label": "Qwen3-VL-235B", "family": "Multimodal & Encoders", "outcome": "ok", "detail": null, "valid": true, "C": 0.9990899221600671, "s": { "blend": 1.1413509933751536, "thr": 1.1380032231909216, "lat": 1.144708612007068 }, "mean_d": 0.025565478802446362, "n_dropped": 1, "as_is": { "outcome": "crash", "detail": "compile", "valid": false } }, { "model": "jameslahm/yolov10n", "label": "YOLOv10n", "family": "Edge & Detection", "outcome": "ok", "detail": null, "valid": true, "C": 0.9973237955792069, "s": { "blend": 1.0136153509660921, "thr": 1.1890509877048334, "lat": 0.8640639386686728 }, "mean_d": 0.002050683802241063, "n_dropped": 2, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "OpenFold/OpenFold3", "label": "OpenFold3", "family": "3D / Robotics / Science", "outcome": "ok", "detail": null, "valid": true, "C": 0.9874953195835267, "s": { "blend": 0.7209618183429709, "thr": 0.6158064010602652, "lat": 0.8440736286817758 }, "mean_d": 0.12909876524023348, "n_dropped": 0, "as_is": { "outcome": "ok", "detail": null, "valid": true } }, { "model": "BAAI/bge-m3", "label": "BGE-M3", "family": "Recommendation & Specialized", "outcome": "ok", "detail": null, "valid": true, "C": 0.9999992253051744, "s": { "blend": 1.0724222808846824, "thr": 1.071721979079743, "lat": 1.073123040292085 }, "mean_d": 6.189185797944412e-05, "n_dropped": 0, "as_is": { "outcome": "ok", "detail": null, "valid": true } }, { "model": "Etched/oasis-500m", "label": "Oasis-500M", "family": "World Models", "outcome": "ok", "detail": null, "valid": true, "C": 0.9231423690991811, "s": { "blend": 1.1803179543573088, "thr": 1.1381346204919784, "lat": 1.224064753232802 }, "mean_d": 0.0393519578195086, "n_dropped": 0, "as_is": { "outcome": "ok", "detail": null, "valid": true } } ], "sensitivity": { "0.5x": { "score": 0.7564574372387555, "rank": 2, "of": 10 }, "1x": { "score": 1.026229498257274, "rank": 2, "of": 10 }, "2x": { "score": 1.0364749147734484, "rank": 2, "of": 10 } } }, "cost": { "tokens": 9477354756, "tokens_output": 42154800, "tokens_cached": 9166880046, "n_ops": 86, "tokens_per_op": 110201799.48837209, "agent_hours": 761.528888888889, "agent_seconds": 2741504.0000000005, "median_hours_per_op": 8.217502777777778, "source": "agent_cost.json" }, "provenance": { "run_dir": "kda-fk-runs", "bench_files": 3, "bench_sha256": "a59e900f08782b78" } }, { "agent": "AKO", "agent_key": "ako", "mode": "ind", "mode_label": "Independent", "open_weights": null, "kernel": { "L1": { "benched": 45, "correct": 45, "geomean": 1.8419763424286186, "fast_1": 44, "fast_2": 11 }, "L2": { "benched": 64, "correct": 64, "geomean": 3.345055851267434, "fast_1": 64, "fast_2": 41 }, "L3": { "benched": 21, "correct": 18, "geomean": 6.475416087212871, "fast_1": 18, "fast_2": 17 } }, "composition": { "ind_to_comp_L2": { "n": 60, "from": 3.5557514033605613, "to": 3.5444349131138506, "ratio": 0.9968174124220227, "broke": 4 }, "ind_to_comp_L3": { "n": 14, "from": 7.619490368745697, "to": 6.589243041481398, "ratio": 0.864787895593351, "broke": 3 }, "comp_to_seq_L2": { "n": 55, "from": 3.4603537447778097, "to": 3.6469806374212435, "ratio": 1.0539328942669755, "broke": 1 }, "comp_to_seq_L3": { "n": 14, "from": 6.589243041481398, "to": 9.722389869804253, "ratio": 1.4754941969204491, "broke": 0 } }, "e2e": { "deployable": { "n_items": 11, "n_valid": 7, "coverage_macro": 0.5625, "C_macro": 0.6525215490922178, "S_macro": { "blend": 1.1460186472724327, "thr": 1.1324606572846503, "lat": 1.159738955563572 }, "score": { "blend": 0.4206385479413114, "thr": 0.4156621775872713, "lat": 0.4256744961526497 }, "n_families": 8, "n_families_valid": 5, "families": { "Dense & MoE LLMs": { "n": 2, "n_valid": 1, "C_f": 0.49991697788559475, "coverage_f": 0.5, "S_f": { "blend": 1.0216028624839093, "thr": 1.0246325835413959, "lat": 1.018582099964179 } }, "Linear Attention & New Archs": { "n": 3, "n_valid": 3, "C_f": 0.9864211139224645, "coverage_f": 1.0, "S_f": { "blend": 1.0698879433650113, "thr": 1.0537005667923838, "lat": 1.0863239969988099 } }, "Vision / Video / Audio": { "n": 1, "n_valid": 0, "C_f": 0.7395800103965621, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Multimodal & Encoders": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Edge & Detection": { "n": 1, "n_valid": 1, "C_f": 0.9954476195073725, "coverage_f": 1.0, "S_f": { "blend": 1.3419076068664664, "thr": 1.3184204434389202, "lat": 1.3658131852606623 } }, "3D / Robotics / Science": { "n": 1, "n_valid": 1, "C_f": 0.9989266142512425, "coverage_f": 1.0, "S_f": { "blend": 1.4789529363035305, "thr": 1.2693460518876172, "lat": 1.7231721677064704 } }, "Recommendation & Specialized": { "n": 1, "n_valid": 1, "C_f": 0.999880056774506, "coverage_f": 1.0, "S_f": { "blend": 0.9112993445414342, "thr": 1.0308518973228313, "lat": 0.8056118415442669 } }, "World Models": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } } } }, "all_winners": { "n_items": 11, "n_valid": 1, "coverage_macro": 0.125, "C_macro": 0.2174325083963835, "S_macro": { "blend": 0.9112993445414342, "thr": 1.0308518973228313, "lat": 0.8056118415442669 }, "score": { "blend": 0.024768262797953022, "thr": 0.0280175892275093, "lat": 0.021895775437599973 }, "n_families": 8, "n_families_valid": 1, "families": { "Dense & MoE LLMs": { "n": 2, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Linear Attention & New Archs": { "n": 3, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Vision / Video / Audio": { "n": 1, "n_valid": 0, "C_f": 0.7395800103965621, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Multimodal & Encoders": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Edge & Detection": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "3D / Robotics / Science": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Recommendation & Specialized": { "n": 1, "n_valid": 1, "C_f": 0.999880056774506, "coverage_f": 1.0, "S_f": { "blend": 0.9112993445414342, "thr": 1.0308518973228313, "lat": 0.8056118415442669 } }, "World Models": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } } } }, "drops": { "requested": 190, "kept": 117, "kept_fraction": 0.6157894736842106, "dropped_unique_kernels": 15, "dropped_by_category": { "runtime": 14, "timeout": 1, "compile": 1, "incorrect": 3 }, "status": { "ok": 8, "incomplete": 3 } }, "per_model": [ { "model": "meta-llama/Llama-3.1-8B-Instruct", "label": "Llama-3.1-8B", "family": "Dense & MoE LLMs", "outcome": "ok", "detail": null, "valid": true, "C": 0.9998339557711895, "s": { "blend": 1.0216028624839093, "thr": 1.0246325835413959, "lat": 1.018582099964179 }, "mean_d": 0.005644893730402415, "n_dropped": 2, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "openai/gpt-oss-120b", "label": "GPT-OSS-120B", "family": "Dense & MoE LLMs", "outcome": "incomplete", "detail": null, "valid": false, "C": 0.0, "s": { "blend": null, "thr": null, "lat": null }, "mean_d": null, "n_dropped": 3, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "Qwen/Qwen3-Next-80B-A3B-Instruct", "label": "Qwen3-Next-80B", "family": "Linear Attention & New Archs", "outcome": "ok", "detail": null, "valid": true, "C": 0.9640421313043508, "s": { "blend": 1.020310528624325, "thr": 1.0344702433519621, "lat": 1.006344630511962 }, "mean_d": 0.018802351479129058, "n_dropped": 3, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "moonshotai/Kimi-Linear-48B-A3B-Instruct", "label": "Kimi-Linear-48B", "family": "Linear Attention & New Archs", "outcome": "ok", "detail": null, "valid": true, "C": 0.9963537489589973, "s": { "blend": 1.0351758417741455, "thr": 0.9949207663922159, "lat": 1.0770596610206558 }, "mean_d": 0.02294267837107974, "n_dropped": 3, "as_is": { "outcome": "crash", "detail": "compile", "valid": false } }, { "model": "fla-hub/gla-2.7B-100B", "label": "GLA-2.7B", "family": "Linear Attention & New Archs", "outcome": "ok", "detail": null, "valid": true, "C": 0.9988674615040453, "s": { "blend": 1.15949367524863, "thr": 1.1366981223220403, "lat": 1.1827463743805529 }, "mean_d": 0.0007526367094814433, "n_dropped": 2, "as_is": { "outcome": "broken", "detail": null, "valid": false } }, { "model": "black-forest-labs/FLUX.1-dev", "label": "FLUX.1-dev", "family": "Vision / Video / Audio", "outcome": "ok", "detail": null, "valid": false, "C": 0.7395800103965621, "s": { "blend": 1.2889617492498917, "thr": 1.2359562692290744, "lat": 1.34424043341408 }, "mean_d": 0.13244508578810088, "n_dropped": 0, "as_is": { "outcome": "ok", "detail": null, "valid": false } }, { "model": "Qwen/Qwen3-VL-235B-A22B-Instruct-FP8", "label": "Qwen3-VL-235B", "family": "Multimodal & Encoders", "outcome": "incomplete", "detail": null, "valid": false, "C": 0.0, "s": { "blend": null, "thr": null, "lat": null }, "mean_d": null, "n_dropped": 1, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "jameslahm/yolov10n", "label": "YOLOv10n", "family": "Edge & Detection", "outcome": "ok", "detail": null, "valid": true, "C": 0.9954476195073725, "s": { "blend": 1.3419076068664664, "thr": 1.3184204434389202, "lat": 1.3658131852606623 }, "mean_d": 0.002981973530443086, "n_dropped": 4, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "OpenFold/OpenFold3", "label": "OpenFold3", "family": "3D / Robotics / Science", "outcome": "ok", "detail": null, "valid": true, "C": 0.9989266142512425, "s": { "blend": 1.4789529363035305, "thr": 1.2693460518876172, "lat": 1.7231721677064704 }, "mean_d": 0.11332063120306418, "n_dropped": 1, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "BAAI/bge-m3", "label": "BGE-M3", "family": "Recommendation & Specialized", "outcome": "ok", "detail": null, "valid": true, "C": 0.999880056774506, "s": { "blend": 0.9112993445414342, "thr": 1.0308518973228313, "lat": 0.8056118415442669 }, "mean_d": 0.00025382946442956134, "n_dropped": 0, "as_is": { "outcome": "ok", "detail": null, "valid": true } }, { "model": "Etched/oasis-500m", "label": "Oasis-500M", "family": "World Models", "outcome": "incomplete", "detail": null, "valid": false, "C": 0.0, "s": { "blend": null, "thr": null, "lat": null }, "mean_d": null, "n_dropped": 0, "as_is": { "outcome": "broken", "detail": null, "valid": false } } ], "sensitivity": { "0.5x": { "score": 0.2801416680271853, "rank": 6, "of": 10 }, "1x": { "score": 0.4206385479413114, "rank": 7, "of": 10 }, "2x": { "score": 0.4320191295983204, "rank": 8, "of": 10 } } }, "cost": { "tokens": 5466371449, "tokens_output": 47442291, "tokens_cached": 5296912450, "n_ops": 133, "tokens_per_op": 41100537.21052632, "agent_hours": 1420.5569496379803, "agent_seconds": 5114005.018696729, "median_hours_per_op": 9.432597222222222, "source": "agent_cost.json" }, "provenance": { "run_dir": "ako4x-fk-runs", "bench_files": 3, "bench_sha256": "b0bc86f1c706021a" } }, { "agent": "AKO", "agent_key": "ako", "mode": "comp", "mode_label": "Composed", "open_weights": null, "kernel": { "L1": { "benched": 45, "correct": 45, "geomean": 1.866771340394484, "fast_1": 44, "fast_2": 11 }, "L2": { "benched": 64, "correct": 60, "geomean": 3.5444349131138506, "fast_1": 60, "fast_2": 43 }, "L3": { "benched": 17, "correct": 14, "geomean": 6.589243041481398, "fast_1": 14, "fast_2": 13 } }, "composition": { "ind_to_comp_L2": { "n": 60, "from": 3.5557514033605613, "to": 3.5444349131138506, "ratio": 0.9968174124220227, "broke": 4 }, "ind_to_comp_L3": { "n": 14, "from": 7.619490368745697, "to": 6.589243041481398, "ratio": 0.864787895593351, "broke": 3 }, "comp_to_seq_L2": { "n": 55, "from": 3.4603537447778097, "to": 3.6469806374212435, "ratio": 1.0539328942669755, "broke": 1 }, "comp_to_seq_L3": { "n": 14, "from": 6.589243041481398, "to": 9.722389869804253, "ratio": 1.4754941969204491, "broke": 0 } }, "e2e": null, "cost": { "tokens": 5466371449, "tokens_output": 47442291, "tokens_cached": 5296912450, "n_ops": 133, "tokens_per_op": 41100537.21052632, "agent_hours": 1420.5569496379803, "agent_seconds": 5114005.018696729, "median_hours_per_op": 9.432597222222222, "source": "agent_cost.json" }, "provenance": { "run_dir": "ako4x-fk-runs", "bench_files": 3, "bench_sha256": "b0bc86f1c706021a" } }, { "agent": "AKO", "agent_key": "ako", "mode": "seq", "mode_label": "Sequential", "open_weights": null, "kernel": { "L1": { "benched": 45, "correct": 45, "geomean": 1.8628030778015945, "fast_1": 44, "fast_2": 11 }, "L2": { "benched": 60, "correct": 59, "geomean": 3.514020786850813, "fast_1": 59, "fast_2": 40 }, "L3": { "benched": 20, "correct": 20, "geomean": 6.54802198796024, "fast_1": 20, "fast_2": 17 } }, "composition": { "ind_to_comp_L2": { "n": 60, "from": 3.5557514033605613, "to": 3.5444349131138506, "ratio": 0.9968174124220227, "broke": 4 }, "ind_to_comp_L3": { "n": 14, "from": 7.619490368745697, "to": 6.589243041481398, "ratio": 0.864787895593351, "broke": 3 }, "comp_to_seq_L2": { "n": 55, "from": 3.4603537447778097, "to": 3.6469806374212435, "ratio": 1.0539328942669755, "broke": 1 }, "comp_to_seq_L3": { "n": 14, "from": 6.589243041481398, "to": 9.722389869804253, "ratio": 1.4754941969204491, "broke": 0 } }, "e2e": { "deployable": { "n_items": 11, "n_valid": 9, "coverage_macro": 0.8333333333333333, "C_macro": 0.8167162987734081, "S_macro": { "blend": 1.2455726460944592, "thr": 1.0790055818565667, "lat": 1.4378528181747523 }, "score": { "blend": 0.8477329011430556, "thr": 0.7343678709747856, "lat": 0.9785981932004981 }, "n_families": 8, "n_families_valid": 7, "families": { "Dense & MoE LLMs": { "n": 2, "n_valid": 2, "C_f": 0.9883640021937643, "coverage_f": 1.0, "S_f": { "blend": 0.9913967391637554, "thr": 1.0109335527196395, "lat": 0.9722374846303119 } }, "Linear Attention & New Archs": { "n": 3, "n_valid": 2, "C_f": 0.6524795063110135, "coverage_f": 0.6666666666666666, "S_f": { "blend": 1.1462637603275991, "thr": 1.1054645791561142, "lat": 1.1885687094953181 } }, "Vision / Video / Audio": { "n": 1, "n_valid": 1, "C_f": 0.977293726010436, "coverage_f": 1.0, "S_f": { "blend": 1.3272851464652933, "thr": 1.179653883746554, "lat": 1.4933921587511085 } }, "Multimodal & Encoders": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Edge & Detection": { "n": 1, "n_valid": 1, "C_f": 0.9950881936778813, "coverage_f": 1.0, "S_f": { "blend": 2.1683546233213176, "thr": 1.320224624082015, "lat": 3.5613347052575874 } }, "3D / Robotics / Science": { "n": 1, "n_valid": 1, "C_f": 0.9904569318831078, "coverage_f": 1.0, "S_f": { "blend": 1.3631177799292182, "thr": 1.2542043872784963, "lat": 1.4814890625530648 } }, "Recommendation & Specialized": { "n": 1, "n_valid": 1, "C_f": 0.9999838586435781, "coverage_f": 1.0, "S_f": { "blend": 1.0238694442289749, "thr": 0.9630675217634831, "lat": 1.0885100111217338 } }, "World Models": { "n": 1, "n_valid": 1, "C_f": 0.9300641714674848, "coverage_f": 1.0, "S_f": { "blend": 1.0190110120899152, "thr": 0.8099736928442993, "lat": 1.281996504249578 } } } }, "all_winners": { "n_items": 11, "n_valid": 2, "coverage_macro": 0.25, "C_macro": 0.24938400654018242, "S_macro": { "blend": 1.490004041293622, "thr": 1.1275927708555924, "lat": 1.9688952434368248 }, "score": { "blend": 0.09289579439471671, "thr": 0.07030090073542837, "lat": 0.12275274606654579 }, "n_families": 8, "n_families_valid": 2, "families": { "Dense & MoE LLMs": { "n": 2, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Linear Attention & New Archs": { "n": 3, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Vision / Video / Audio": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Multimodal & Encoders": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Edge & Detection": { "n": 1, "n_valid": 1, "C_f": 0.9950881936778813, "coverage_f": 1.0, "S_f": { "blend": 2.1683546233213176, "thr": 1.320224624082015, "lat": 3.5613347052575874 } }, "3D / Robotics / Science": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Recommendation & Specialized": { "n": 1, "n_valid": 1, "C_f": 0.9999838586435781, "coverage_f": 1.0, "S_f": { "blend": 1.0238694442289749, "thr": 0.9630675217634831, "lat": 1.0885100111217338 } }, "World Models": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } } } }, "drops": { "requested": 201, "kept": 122, "kept_fraction": 0.6069651741293532, "dropped_unique_kernels": 26, "dropped_by_category": { "compile": 14, "runtime": 19, "incorrect": 2, "import": 1 }, "status": { "ok": 9, "incomplete": 2 } }, "per_model": [ { "model": "meta-llama/Llama-3.1-8B-Instruct", "label": "Llama-3.1-8B", "family": "Dense & MoE LLMs", "outcome": "ok", "detail": null, "valid": true, "C": 1.0, "s": { "blend": 1.0196084491899402, "thr": 1.018988568821035, "lat": 1.0202287066500941 }, "mean_d": 0.0, "n_dropped": 6, "as_is": { "outcome": "crash", "detail": "compile", "valid": false } }, { "model": "openai/gpt-oss-120b", "label": "GPT-OSS-120B", "family": "Dense & MoE LLMs", "outcome": "ok", "detail": null, "valid": true, "C": 0.9767280043875285, "s": { "blend": 0.9639656234757538, "thr": 1.002942210820663, "lat": 0.9265037538729686 }, "mean_d": 0.012253350810623273, "n_dropped": 7, "as_is": { "outcome": "crash", "detail": "compile", "valid": false } }, { "model": "Qwen/Qwen3-Next-80B-A3B-Instruct", "label": "Qwen3-Next-80B", "family": "Linear Attention & New Archs", "outcome": "ok", "detail": null, "valid": true, "C": 0.96415818012206, "s": { "blend": 1.0649742528031894, "thr": 1.0479310239880073, "lat": 1.0822946674653382 }, "mean_d": 0.018775693119091982, "n_dropped": 4, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "moonshotai/Kimi-Linear-48B-A3B-Instruct", "label": "Kimi-Linear-48B", "family": "Linear Attention & New Archs", "outcome": "incomplete", "detail": null, "valid": false, "C": 0.0, "s": { "blend": null, "thr": null, "lat": null }, "mean_d": null, "n_dropped": 5, "as_is": { "outcome": "crash", "detail": "compile", "valid": false } }, { "model": "fla-hub/gla-2.7B-100B", "label": "GLA-2.7B", "family": "Linear Attention & New Archs", "outcome": "ok", "detail": null, "valid": true, "C": 0.9932803388109805, "s": { "blend": 1.2337580977022777, "thr": 1.166156844100447, "lat": 1.305278146200059 }, "mean_d": 0.003602378511445639, "n_dropped": 3, "as_is": { "outcome": "broken", "detail": null, "valid": false } }, { "model": "black-forest-labs/FLUX.1-dev", "label": "FLUX.1-dev", "family": "Vision / Video / Audio", "outcome": "ok", "detail": null, "valid": true, "C": 0.977293726010436, "s": { "blend": 1.3272851464652933, "thr": 1.179653883746554, "lat": 1.4933921587511085 }, "mean_d": 0.012239088888580519, "n_dropped": 2, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "Qwen/Qwen3-VL-235B-A22B-Instruct-FP8", "label": "Qwen3-VL-235B", "family": "Multimodal & Encoders", "outcome": "incomplete", "detail": null, "valid": false, "C": 0.0, "s": { "blend": null, "thr": null, "lat": null }, "mean_d": null, "n_dropped": 3, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "jameslahm/yolov10n", "label": "YOLOv10n", "family": "Edge & Detection", "outcome": "ok", "detail": null, "valid": true, "C": 0.9950881936778813, "s": { "blend": 2.1683546233213176, "thr": 1.320224624082015, "lat": 3.5613347052575874 }, "mean_d": 0.0031144166296152506, "n_dropped": 0, "as_is": { "outcome": "ok", "detail": null, "valid": true } }, { "model": "OpenFold/OpenFold3", "label": "OpenFold3", "family": "3D / Robotics / Science", "outcome": "ok", "detail": null, "valid": true, "C": 0.9904569318831078, "s": { "blend": 1.3631177799292182, "thr": 1.2542043872784963, "lat": 1.4814890625530648 }, "mean_d": 0.1299574628467756, "n_dropped": 5, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "BAAI/bge-m3", "label": "BGE-M3", "family": "Recommendation & Specialized", "outcome": "ok", "detail": null, "valid": true, "C": 0.9999838586435781, "s": { "blend": 1.0238694442289749, "thr": 0.9630675217634831, "lat": 1.0885100111217338 }, "mean_d": 7.717673107026854e-05, "n_dropped": 0, "as_is": { "outcome": "ok", "detail": null, "valid": true } }, { "model": "Etched/oasis-500m", "label": "Oasis-500M", "family": "World Models", "outcome": "ok", "detail": null, "valid": true, "C": 0.9300641714674848, "s": { "blend": 1.0190110120899152, "thr": 0.8099736928442993, "lat": 1.281996504249578 }, "mean_d": 0.03589797843772504, "n_dropped": 1, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } } ], "sensitivity": { "0.5x": { "score": 0.5676691163432587, "rank": 3, "of": 10 }, "1x": { "score": 0.8477329011430556, "rank": 3, "of": 10 }, "2x": { "score": 0.8573226242320704, "rank": 3, "of": 10 } } }, "cost": { "tokens": 4860846596, "tokens_output": 35309068, "tokens_cached": 4724611179, "n_ops": 86, "tokens_per_op": 56521472.04651163, "agent_hours": 813.1366888888884, "agent_seconds": 2927292.079999998, "median_hours_per_op": 8.703480555555556, "source": "agent_cost.json" }, "provenance": { "run_dir": "ako4x-fk-runs", "bench_files": 3, "bench_sha256": "b0bc86f1c706021a" } }, { "agent": "Codex", "agent_key": "codex", "mode": "ind", "mode_label": "Independent", "open_weights": false, "kernel": { "L1": { "benched": 47, "correct": 47, "geomean": 1.6160615026333696, "fast_1": 40, "fast_2": 9 }, "L2": { "benched": 64, "correct": 64, "geomean": 2.8746749849442086, "fast_1": 60, "fast_2": 39 }, "L3": { "benched": 21, "correct": 19, "geomean": 2.9235058654116415, "fast_1": 18, "fast_2": 12 } }, "composition": { "ind_to_comp_L2": { "n": 47, "from": 3.637702121971566, "to": 3.5188419994832243, "ratio": 0.9673254932638844, "broke": 13 }, "ind_to_comp_L3": { "n": 5, "from": 5.552907573427215, "to": 5.791804132956942, "ratio": 1.043021886528949, "broke": 13 }, "comp_to_seq_L2": { "n": 43, "from": 3.559827276629477, "to": 3.3639471944587847, "ratio": 0.944974835308258, "broke": 1 }, "comp_to_seq_L3": { "n": 5, "from": 5.791804132956942, "to": 7.798545675628716, "ratio": 1.3464795246187398, "broke": 0 } }, "e2e": { "deployable": { "n_items": 11, "n_valid": 5, "coverage_macro": 0.3333333333333333, "C_macro": 0.33005982994287153, "S_macro": { "blend": 0.9567785493179931, "thr": 0.975659591453057, "lat": 0.938262894614395 }, "score": { "blend": 0.1052647217602947, "thr": 0.10734201294570916, "lat": 0.10322763047937786 }, "n_families": 8, "n_families_valid": 3, "families": { "Dense & MoE LLMs": { "n": 2, "n_valid": 2, "C_f": 0.9998176219729005, "coverage_f": 1.0, "S_f": { "blend": 0.9969881376419626, "thr": 0.9835524992185045, "lat": 1.0106073111385248 } }, "Linear Attention & New Archs": { "n": 3, "n_valid": 2, "C_f": 0.6494174489867307, "coverage_f": 0.6666666666666666, "S_f": { "blend": 0.8213050830797274, "thr": 0.8675198236053437, "lat": 0.7775523061700821 } }, "Vision / Video / Audio": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Multimodal & Encoders": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Edge & Detection": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "3D / Robotics / Science": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Recommendation & Specialized": { "n": 1, "n_valid": 1, "C_f": 0.9912435685833411, "coverage_f": 1.0, "S_f": { "blend": 1.0696453026294144, "thr": 1.0884738533837883, "lat": 1.051142450395412 } }, "World Models": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } } } }, "all_winners": { "n_items": 11, "n_valid": 1, "coverage_macro": 0.125, "C_macro": 0.12390544607291763, "S_macro": { "blend": 1.0696453026294144, "thr": 1.0884738533837883, "lat": 1.051142450395412 }, "score": { "blend": 0.01656685979526232, "thr": 0.01685847979277823, "lat": 0.016280284275302905 }, "n_families": 8, "n_families_valid": 1, "families": { "Dense & MoE LLMs": { "n": 2, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Linear Attention & New Archs": { "n": 3, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Vision / Video / Audio": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Multimodal & Encoders": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Edge & Detection": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "3D / Robotics / Science": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Recommendation & Specialized": { "n": 1, "n_valid": 1, "C_f": 0.9912435685833411, "coverage_f": 1.0, "S_f": { "blend": 1.0696453026294144, "thr": 1.0884738533837883, "lat": 1.051142450395412 } }, "World Models": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } } } }, "drops": { "requested": 153, "kept": 33, "kept_fraction": 0.21568627450980393, "dropped_unique_kernels": 29, "dropped_by_category": { "compile": 9, "runtime": 23, "incorrect": 4 }, "status": { "ok": 5, "incomplete": 6 } }, "per_model": [ { "model": "meta-llama/Llama-3.1-8B-Instruct", "label": "Llama-3.1-8B", "family": "Dense & MoE LLMs", "outcome": "ok", "detail": null, "valid": true, "C": 0.9996352439458009, "s": { "blend": 0.9935497775959187, "thr": 0.9829351163687577, "lat": 1.004279065954709 }, "mean_d": 0.00400392113698346, "n_dropped": 2, "as_is": { "outcome": "crash", "detail": "compile", "valid": false } }, { "model": "openai/gpt-oss-120b", "label": "GPT-OSS-120B", "family": "Dense & MoE LLMs", "outcome": "ok", "detail": null, "valid": true, "C": 1.0, "s": { "blend": 1.00043839675947, "thr": 0.9841702698472377, "lat": 1.0169754323771785 }, "mean_d": 0.0, "n_dropped": 4, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "Qwen/Qwen3-Next-80B-A3B-Instruct", "label": "Qwen3-Next-80B", "family": "Linear Attention & New Archs", "outcome": "ok", "detail": null, "valid": true, "C": 0.9645019515222369, "s": { "blend": 1.0261845057010959, "thr": 1.0109637997979748, "lat": 1.0416343690559828 }, "mean_d": 0.018619776190403725, "n_dropped": 3, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "moonshotai/Kimi-Linear-48B-A3B-Instruct", "label": "Kimi-Linear-48B", "family": "Linear Attention & New Archs", "outcome": "incomplete", "detail": null, "valid": false, "C": 0.0, "s": { "blend": null, "thr": null, "lat": null }, "mean_d": null, "n_dropped": 5, "as_is": { "outcome": "crash", "detail": "compile", "valid": false } }, { "model": "fla-hub/gla-2.7B-100B", "label": "GLA-2.7B", "family": "Linear Attention & New Archs", "outcome": "ok", "detail": null, "valid": true, "C": 0.9837503954379553, "s": { "blend": 0.657330173808994, "thr": 0.7444288751967579, "lat": 0.5804220816737656 }, "mean_d": 0.008826570254585166, "n_dropped": 2, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "black-forest-labs/FLUX.1-dev", "label": "FLUX.1-dev", "family": "Vision / Video / Audio", "outcome": "incomplete", "detail": null, "valid": false, "C": 0.0, "s": { "blend": null, "thr": null, "lat": null }, "mean_d": null, "n_dropped": 4, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "Qwen/Qwen3-VL-235B-A22B-Instruct-FP8", "label": "Qwen3-VL-235B", "family": "Multimodal & Encoders", "outcome": "incomplete", "detail": null, "valid": false, "C": 0.0, "s": { "blend": null, "thr": null, "lat": null }, "mean_d": null, "n_dropped": 5, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "jameslahm/yolov10n", "label": "YOLOv10n", "family": "Edge & Detection", "outcome": "incomplete", "detail": null, "valid": false, "C": 0.0, "s": { "blend": null, "thr": null, "lat": null }, "mean_d": null, "n_dropped": 5, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "OpenFold/OpenFold3", "label": "OpenFold3", "family": "3D / Robotics / Science", "outcome": "incomplete", "detail": null, "valid": false, "C": 0.0, "s": { "blend": null, "thr": null, "lat": null }, "mean_d": null, "n_dropped": 3, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "BAAI/bge-m3", "label": "BGE-M3", "family": "Recommendation & Specialized", "outcome": "ok", "detail": null, "valid": true, "C": 0.9912435685833411, "s": { "blend": 1.0696453026294144, "thr": 1.0884738533837883, "lat": 1.051142450395412 }, "mean_d": 0.005369459276912864, "n_dropped": 0, "as_is": { "outcome": "ok", "detail": null, "valid": true } }, { "model": "Etched/oasis-500m", "label": "Oasis-500M", "family": "World Models", "outcome": "incomplete", "detail": null, "valid": false, "C": 0.0, "s": { "blend": null, "thr": null, "lat": null }, "mean_d": null, "n_dropped": 3, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } } ], "sensitivity": { "0.5x": { "score": 0.10404783983760925, "rank": 10, "of": 10 }, "1x": { "score": 0.1052647217602947, "rank": 10, "of": 10 }, "2x": { "score": 0.10585139561379542, "rank": 10, "of": 10 } } }, "cost": { "tokens": 1023445157, "tokens_output": 3983123, "tokens_cached": 998700406, "n_ops": 133, "tokens_per_op": 7695076.368421053, "agent_hours": 253.4856722222222, "agent_seconds": 912548.4199999999, "median_hours_per_op": 1.8553111111111111, "source": "agent_cost.json" }, "provenance": { "run_dir": "codex-fk-runs", "bench_files": 3, "bench_sha256": "9248edeabf758364" } }, { "agent": "Codex", "agent_key": "codex", "mode": "comp", "mode_label": "Composed", "open_weights": false, "kernel": { "L1": { "benched": 41, "correct": 41, "geomean": 1.757753166739466, "fast_1": 40, "fast_2": 9 }, "L2": { "benched": 60, "correct": 47, "geomean": 3.5188419994832243, "fast_1": 45, "fast_2": 33 }, "L3": { "benched": 18, "correct": 5, "geomean": 5.791804132956942, "fast_1": 5, "fast_2": 4 } }, "composition": { "ind_to_comp_L2": { "n": 47, "from": 3.637702121971566, "to": 3.5188419994832243, "ratio": 0.9673254932638844, "broke": 13 }, "ind_to_comp_L3": { "n": 5, "from": 5.552907573427215, "to": 5.791804132956942, "ratio": 1.043021886528949, "broke": 13 }, "comp_to_seq_L2": { "n": 43, "from": 3.559827276629477, "to": 3.3639471944587847, "ratio": 0.944974835308258, "broke": 1 }, "comp_to_seq_L3": { "n": 5, "from": 5.791804132956942, "to": 7.798545675628716, "ratio": 1.3464795246187398, "broke": 0 } }, "e2e": null, "cost": { "tokens": 1023445157, "tokens_output": 3983123, "tokens_cached": 998700406, "n_ops": 133, "tokens_per_op": 7695076.368421053, "agent_hours": 253.4856722222222, "agent_seconds": 912548.4199999999, "median_hours_per_op": 1.8553111111111111, "source": "agent_cost.json" }, "provenance": { "run_dir": "codex-fk-runs", "bench_files": 3, "bench_sha256": "9248edeabf758364" } }, { "agent": "Codex", "agent_key": "codex", "mode": "seq", "mode_label": "Sequential", "open_weights": false, "kernel": { "L1": { "benched": 41, "correct": 41, "geomean": 1.753640768959977, "fast_1": 38, "fast_2": 9 }, "L2": { "benched": 57, "correct": 56, "geomean": 2.896102549099732, "fast_1": 56, "fast_2": 34 }, "L3": { "benched": 18, "correct": 18, "geomean": 3.9486818170890046, "fast_1": 18, "fast_2": 11 } }, "composition": { "ind_to_comp_L2": { "n": 47, "from": 3.637702121971566, "to": 3.5188419994832243, "ratio": 0.9673254932638844, "broke": 13 }, "ind_to_comp_L3": { "n": 5, "from": 5.552907573427215, "to": 5.791804132956942, "ratio": 1.043021886528949, "broke": 13 }, "comp_to_seq_L2": { "n": 43, "from": 3.559827276629477, "to": 3.3639471944587847, "ratio": 0.944974835308258, "broke": 1 }, "comp_to_seq_L3": { "n": 5, "from": 5.791804132956942, "to": 7.798545675628716, "ratio": 1.3464795246187398, "broke": 0 } }, "e2e": { "deployable": { "n_items": 11, "n_valid": 8, "coverage_macro": 0.7083333333333333, "C_macro": 0.712375159546902, "S_macro": { "blend": 1.0961109089119632, "thr": 1.1125790457771594, "lat": 1.0798865295872675 }, "score": { "blend": 0.5530965467288921, "thr": 0.5614063532978644, "lat": 0.5449097399884899 }, "n_families": 8, "n_families_valid": 6, "families": { "Dense & MoE LLMs": { "n": 2, "n_valid": 2, "C_f": 0.9998176219729005, "coverage_f": 1.0, "S_f": { "blend": 1.0074432461722211, "thr": 0.9967198286972025, "lat": 1.0182820337633274 } }, "Linear Attention & New Archs": { "n": 3, "n_valid": 2, "C_f": 0.8580854465096683, "coverage_f": 0.6666666666666666, "S_f": { "blend": 0.8126833493238492, "thr": 0.8680689357393532, "lat": 0.760831541225129 } }, "Vision / Video / Audio": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Multimodal & Encoders": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Edge & Detection": { "n": 1, "n_valid": 1, "C_f": 0.9368644826391289, "coverage_f": 1.0, "S_f": { "blend": 1.1967694897662806, "thr": 1.1500180453216584, "lat": 1.24542151095972 } }, "3D / Robotics / Science": { "n": 1, "n_valid": 1, "C_f": 0.9638433303284187, "coverage_f": 1.0, "S_f": { "blend": 1.731650299670688, "thr": 1.6544975278791816, "lat": 1.812400870851319 } }, "Recommendation & Specialized": { "n": 1, "n_valid": 1, "C_f": 0.9911624214198009, "coverage_f": 1.0, "S_f": { "blend": 1.0765227986508192, "thr": 1.074171547617353, "lat": 1.0788791963309592 } }, "World Models": { "n": 1, "n_valid": 1, "C_f": 0.9492279735052987, "coverage_f": 1.0, "S_f": { "blend": 0.9494915753892464, "thr": 1.0725400584499334, "lat": 0.8405599815433256 } } } }, "all_winners": { "n_items": 11, "n_valid": 1, "coverage_macro": 0.125, "C_macro": 0.12389530267747512, "S_macro": { "blend": 1.0765227986508192, "thr": 1.074171547617353, "lat": 1.0788791963309592 }, "score": { "blend": 0.016672014747255732, "thr": 0.01663560112744798, "lat": 0.01670850807273191 }, "n_families": 8, "n_families_valid": 1, "families": { "Dense & MoE LLMs": { "n": 2, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Linear Attention & New Archs": { "n": 3, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Vision / Video / Audio": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Multimodal & Encoders": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Edge & Detection": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "3D / Robotics / Science": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } }, "Recommendation & Specialized": { "n": 1, "n_valid": 1, "C_f": 0.9911624214198009, "coverage_f": 1.0, "S_f": { "blend": 1.0765227986508192, "thr": 1.074171547617353, "lat": 1.0788791963309592 } }, "World Models": { "n": 1, "n_valid": 0, "C_f": 0.0, "coverage_f": 0.0, "S_f": { "blend": null, "thr": null, "lat": null } } } }, "drops": { "requested": 180, "kept": 96, "kept_fraction": 0.5333333333333333, "dropped_unique_kernels": 37, "dropped_by_category": { "compile": 14, "runtime": 24, "incorrect": 4, "import": 2 }, "status": { "ok": 9, "incomplete": 2 } }, "per_model": [ { "model": "meta-llama/Llama-3.1-8B-Instruct", "label": "Llama-3.1-8B", "family": "Dense & MoE LLMs", "outcome": "ok", "detail": null, "valid": true, "C": 0.9996352439458009, "s": { "blend": 1.0116126861255381, "thr": 1.0066091139257451, "lat": 1.0166411296824571 }, "mean_d": 0.00400392113698346, "n_dropped": 3, "as_is": { "outcome": "crash", "detail": "compile", "valid": false } }, { "model": "openai/gpt-oss-120b", "label": "GPT-OSS-120B", "family": "Dense & MoE LLMs", "outcome": "ok", "detail": null, "valid": true, "C": 1.0, "s": { "blend": 1.003290990888257, "thr": 0.9869276993169218, "lat": 1.0199255863364964 }, "mean_d": 0.0, "n_dropped": 5, "as_is": { "outcome": "crash", "detail": "compile", "valid": false } }, { "model": "Qwen/Qwen3-Next-80B-A3B-Instruct", "label": "Qwen3-Next-80B", "family": "Linear Attention & New Archs", "outcome": "ok", "detail": null, "valid": false, "C": 0.5849858531001381, "s": { "blend": 1.1076583048775333, "thr": 1.1549715832050858, "lat": 1.0622832095656083 }, "mean_d": 0.3558521553633594, "n_dropped": 2, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "moonshotai/Kimi-Linear-48B-A3B-Instruct", "label": "Kimi-Linear-48B", "family": "Linear Attention & New Archs", "outcome": "ok", "detail": null, "valid": true, "C": 0.9986179032351409, "s": { "blend": 0.9815400572277917, "thr": 0.990820603883092, "lat": 0.9723464370512946 }, "mean_d": 0.013136831607340405, "n_dropped": 9, "as_is": { "outcome": "crash", "detail": "compile", "valid": false } }, { "model": "fla-hub/gla-2.7B-100B", "label": "GLA-2.7B", "family": "Linear Attention & New Archs", "outcome": "ok", "detail": null, "valid": true, "C": 0.9906525831937261, "s": { "blend": 0.672875468917265, "thr": 0.7605248359213215, "lat": 0.5953275623433668 }, "mean_d": 0.005241753564455625, "n_dropped": 2, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "black-forest-labs/FLUX.1-dev", "label": "FLUX.1-dev", "family": "Vision / Video / Audio", "outcome": "incomplete", "detail": null, "valid": false, "C": 0.0, "s": { "blend": null, "thr": null, "lat": null }, "mean_d": null, "n_dropped": 2, "as_is": { "outcome": "broken", "detail": null, "valid": false } }, { "model": "Qwen/Qwen3-VL-235B-A22B-Instruct-FP8", "label": "Qwen3-VL-235B", "family": "Multimodal & Encoders", "outcome": "incomplete", "detail": null, "valid": false, "C": 0.0, "s": { "blend": null, "thr": null, "lat": null }, "mean_d": null, "n_dropped": 5, "as_is": { "outcome": "crash", "detail": "compile", "valid": false } }, { "model": "jameslahm/yolov10n", "label": "YOLOv10n", "family": "Edge & Detection", "outcome": "ok", "detail": null, "valid": true, "C": 0.9368644826391289, "s": { "blend": 1.1967694897662806, "thr": 1.1500180453216584, "lat": 1.24542151095972 }, "mean_d": 0.040270248163074694, "n_dropped": 6, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "OpenFold/OpenFold3", "label": "OpenFold3", "family": "3D / Robotics / Science", "outcome": "ok", "detail": null, "valid": true, "C": 0.9638433303284187, "s": { "blend": 1.731650299670688, "thr": 1.6544975278791816, "lat": 1.812400870851319 }, "mean_d": 0.15736052927992397, "n_dropped": 6, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } }, { "model": "BAAI/bge-m3", "label": "BGE-M3", "family": "Recommendation & Specialized", "outcome": "ok", "detail": null, "valid": true, "C": 0.9911624214198009, "s": { "blend": 1.0765227986508192, "thr": 1.074171547617353, "lat": 1.0788791963309592 }, "mean_d": 0.00540995171151926, "n_dropped": 0, "as_is": { "outcome": "ok", "detail": null, "valid": true } }, { "model": "Etched/oasis-500m", "label": "Oasis-500M", "family": "World Models", "outcome": "ok", "detail": null, "valid": true, "C": 0.9492279735052987, "s": { "blend": 0.9494915753892464, "thr": 1.0725400584499334, "lat": 0.8405599815433256 }, "mean_d": 0.02632280454057312, "n_dropped": 4, "as_is": { "outcome": "crash", "detail": "runtime", "valid": false } } ], "sensitivity": { "0.5x": { "score": 0.20472224409328071, "rank": 9, "of": 10 }, "1x": { "score": 0.5530965467288921, "rank": 6, "of": 10 }, "2x": { "score": 0.5642384710782683, "rank": 5, "of": 10 } } }, "cost": { "tokens": 735408362, "tokens_output": 2421567, "tokens_cached": 723486438, "n_ops": 86, "tokens_per_op": 8551260.023255814, "agent_hours": 67.48039444444443, "agent_seconds": 242929.41999999995, "median_hours_per_op": 0.709251388888889, "source": "agent_cost.json" }, "provenance": { "run_dir": "codex-fk-runs", "bench_files": 3, "bench_sha256": "9248edeabf758364" } } ] }, "models": { "unit": "model", "description": "Base models behind one fixed agent scaffold (vanilla-agents-fastkernels), so the only variable is the model. Ranked by correct-rate x geomean; see docs/leaderboard/MODEL_BOARD.md for the scope.", "metric": "correct_rate * geomean", "status": "out-of-scope-for-v0", "rows": [], "scope_note": { "question": "Whether the model or the scaffold around it does the work. The agent board's Codex row is already one point on this axis -- it is gpt-5.6-sol under exactly this scaffold -- so the board would extend that point rather than open a new axis.", "why_deferred": "The models reachable from this scaffold are three same-family GPT variants plus one Claude, all closed-weights, which is too narrow a spread to rank meaningfully. CommBench already ranks base models on GPU code across an open-weights spread.", "unblocked_by": "An open-weights endpoint the scaffold can reach. The runner and scoring are in place and validated against the agent board, so adding models is a matter of machine time, not engineering." } } }, "known_gaps": { "missing_e2e": [], "note": null } }