{ "schema_version": 1, "environment": "v2_containerized", "hardware": { "name": "RTX PRO 6000 Blackwell Workstation", "sm": "sm_120a", "vram_gb": 96, "peak_bandwidth_gb_s": 1800.0 }, "problems": [ "01_fp8_gemm", "02_kda_cutlass", "03_paged_attention", "05_topk_bitonic", "06_sonic_moe_swiglu", "07_w4a16_gemm" ], "models": [ { "label": "deepseek-claude/deepseek-flash", "harness": "deepseek-claude", "model": "deepseek-flash", "effort": "", "results": { "01_fp8_gemm": { "run_id": "20260910_202114_deepseek-claude_deepseek-flash_01_fp8_gemm", "correct": true, "has_solution": true, "peak_fraction": 0.331, "elapsed_seconds": 5310, "annotation_verdict": "clean", "output_tokens": 239981 }, "02_kda_cutlass": { "run_id": "20260910_202119_deepseek-claude_deepseek-flash_02_kda_cutlass", "correct": true, "has_solution": true, "peak_fraction": 0.0374, "elapsed_seconds": 8144, "annotation_verdict": "clean", "output_tokens": 446987 }, "03_paged_attention": { "run_id": "20260910_202124_deepseek-claude_deepseek-flash_03_paged_attention", "correct": true, "has_solution": true, "peak_fraction": 0.3622, "elapsed_seconds": 8706, "annotation_verdict": "interesting", "output_tokens": 224852 }, "05_topk_bitonic": { "run_id": "20260910_202129_deepseek-claude_deepseek-flash_05_topk_bitonic", "correct": true, "has_solution": true, "peak_fraction": 0.0425, "elapsed_seconds": 9610, "annotation_verdict": "clean", "output_tokens": 526186 }, "06_sonic_moe_swiglu": { "run_id": "20260910_202134_deepseek-claude_deepseek-flash_06_sonic_moe_swiglu", "correct": true, "has_solution": true, "peak_fraction": 0.1081, "elapsed_seconds": 7116, "annotation_verdict": "clean", "output_tokens": 149924 }, "07_w4a16_gemm": { "run_id": "20260910_202139_deepseek-claude_deepseek-flash_07_w4a16_gemm", "correct": true, "has_solution": true, "peak_fraction": 0.2491, "elapsed_seconds": 7977, "annotation_verdict": "clean", "output_tokens": 302459 } }, "pass_count": 6, "total_runs": 6 }, { "label": "deepseek-claude/deepseek-v4-pro", "harness": "deepseek-claude", "model": "deepseek-v4-pro", "effort": "", "results": { "01_fp8_gemm": { "run_id": "20260813_072817_deepseek-claude_deepseek-v4-pro_01_fp8_gemm", "correct": true, "has_solution": true, "peak_fraction": 0.3959, "elapsed_seconds": 7605, "annotation_verdict": "clean", "output_tokens": 162105 }, "02_kda_cutlass": { "run_id": "20260813_072842_deepseek-claude_deepseek-v4-pro_02_kda_cutlass", "correct": true, "has_solution": true, "peak_fraction": 0.0513, "elapsed_seconds": 15421, "annotation_verdict": "clean", "output_tokens": 791398 }, "03_paged_attention": { "run_id": "20260813_072907_deepseek-claude_deepseek-v4-pro_03_paged_attention", "correct": true, "has_solution": true, "peak_fraction": 0.5, "elapsed_seconds": 9607, "annotation_verdict": "clean", "output_tokens": 309910 }, "05_topk_bitonic": { "run_id": "20260813_072932_deepseek-claude_deepseek-v4-pro_05_topk_bitonic", "correct": true, "has_solution": true, "peak_fraction": 0.0935, "elapsed_seconds": 28346, "annotation_verdict": "clean", "output_tokens": 1511557 }, "06_sonic_moe_swiglu": { "run_id": "20260615_140149_deepseek-claude_deepseek-v4-pro_06_sonic_moe_swiglu", "correct": true, "has_solution": true, "peak_fraction": 0.0533, "elapsed_seconds": 5798, "annotation_verdict": "clean", "output_tokens": 186760 }, "07_w4a16_gemm": { "run_id": "20260813_073022_deepseek-claude_deepseek-v4-pro_07_w4a16_gemm", "correct": true, "has_solution": true, "peak_fraction": 0.3103, "elapsed_seconds": 11626, "annotation_verdict": "clean", "output_tokens": 311893 } }, "pass_count": 6, "total_runs": 6 }, { "label": "kinetic-claude/kinetic-0715[1m]", "harness": "kinetic-claude", "model": "kinetic-0715[1m]", "effort": "", "results": { "01_fp8_gemm": { "run_id": "20260716_145443_kinetic-claude_kinetic-0715_1m__01_fp8_gemm", "correct": true, "has_solution": true, "peak_fraction": 0.3529, "elapsed_seconds": 24290, "annotation_verdict": "clean", "output_tokens": 277885 }, "02_kda_cutlass": { "run_id": "20260716_145756_kinetic-claude_kinetic-0715_1m__02_kda_cutlass", "correct": true, "has_solution": true, "peak_fraction": 0.0493, "elapsed_seconds": 30301, "annotation_verdict": "clean", "output_tokens": 435147 }, "03_paged_attention": { "run_id": "20260716_145821_kinetic-claude_kinetic-0715_1m__03_paged_attention", "correct": true, "has_solution": true, "peak_fraction": 0.5811, "elapsed_seconds": 27663, "annotation_verdict": "clean", "output_tokens": 400496 }, "05_topk_bitonic": { "run_id": "20260716_145846_kinetic-claude_kinetic-0715_1m__05_topk_bitonic", "correct": true, "has_solution": true, "peak_fraction": 0.0895, "elapsed_seconds": 29215, "annotation_verdict": "clean", "output_tokens": 451042 }, "06_sonic_moe_swiglu": { "run_id": "20260716_145911_kinetic-claude_kinetic-0715_1m__06_sonic_moe_swiglu", "correct": true, "has_solution": true, "peak_fraction": 0.0329, "elapsed_seconds": 26089, "annotation_verdict": "clean", "output_tokens": 313703 }, "07_w4a16_gemm": { "run_id": "20260716_145936_kinetic-claude_kinetic-0715_1m__07_w4a16_gemm", "correct": true, "has_solution": true, "peak_fraction": 0.027, "elapsed_seconds": 5790, "annotation_verdict": "clean", "output_tokens": 126449 } }, "pass_count": 6, "total_runs": 6 }, { "label": "or-fable/deepseek/deepseek-v4.1-flash-1017", "harness": "or-fable", "model": "deepseek/deepseek-v4.1-flash-1017", "effort": "", "results": { "01_fp8_gemm": { "run_id": "20261017_215207_or-fable_deepseek_deepseek-v4.1-flash-1017_01_fp8_gemm", "correct": true, "has_solution": true, "peak_fraction": 0.3242, "elapsed_seconds": 7617, "annotation_verdict": "clean", "output_tokens": 411719 }, "02_kda_cutlass": { "run_id": "20261017_215237_or-fable_deepseek_deepseek-v4.1-flash-1017_02_kda_cutlass", "correct": true, "has_solution": true, "peak_fraction": 0.0403, "elapsed_seconds": 7262, "annotation_verdict": "clean", "output_tokens": 504542 }, "03_paged_attention": { "run_id": "20261017_215307_or-fable_deepseek_deepseek-v4.1-flash-1017_03_paged_attention", "correct": true, "has_solution": true, "peak_fraction": 0.4586, "elapsed_seconds": 2928, "annotation_verdict": "clean", "output_tokens": 221253 }, "05_topk_bitonic": { "run_id": "20261017_215337_or-fable_deepseek_deepseek-v4.1-flash-1017_05_topk_bitonic", "correct": true, "has_solution": true, "peak_fraction": 0.0897, "elapsed_seconds": 5430, "annotation_verdict": "interesting", "output_tokens": 550506 }, "06_sonic_moe_swiglu": { "run_id": "20261017_215407_or-fable_deepseek_deepseek-v4.1-flash-1017_06_sonic_moe_swiglu", "correct": true, "has_solution": true, "peak_fraction": 0.106, "elapsed_seconds": 4948, "annotation_verdict": "clean", "output_tokens": 200593 }, "07_w4a16_gemm": { "run_id": "20261017_215437_or-fable_deepseek_deepseek-v4.1-flash-1017_07_w4a16_gemm", "correct": true, "has_solution": true, "peak_fraction": 0.2513, "elapsed_seconds": 10836, "annotation_verdict": "interesting", "output_tokens": 450142 } }, "pass_count": 6, "total_runs": 6 }, { "label": "grok/grok-4.6 [xhigh]", "harness": "grok", "model": "grok-4.6", "effort": "xhigh", "results": { "01_fp8_gemm": { "run_id": "20260813_072757_grok_grok-4.6_01_fp8_gemm", "correct": true, "has_solution": true, "peak_fraction": 0.375, "elapsed_seconds": 2935, "annotation_verdict": "clean" }, "02_kda_cutlass": { "run_id": "20260813_072822_grok_grok-4.6_02_kda_cutlass", "correct": true, "has_solution": true, "peak_fraction": 0.0406, "elapsed_seconds": 4953, "annotation_verdict": "clean" }, "03_paged_attention": { "run_id": "20260813_072847_grok_grok-4.6_03_paged_attention", "correct": true, "has_solution": true, "peak_fraction": 0.6217, "elapsed_seconds": 4970, "annotation_verdict": "clean" }, "05_topk_bitonic": { "run_id": "20260813_072912_grok_grok-4.6_05_topk_bitonic", "correct": true, "has_solution": true, "peak_fraction": 0.0368, "elapsed_seconds": 4480, "annotation_verdict": "clean" }, "06_sonic_moe_swiglu": { "run_id": "20260813_072937_grok_grok-4.6_06_sonic_moe_swiglu", "correct": false, "has_solution": true, "peak_fraction": null, "elapsed_seconds": 4454, "annotation_verdict": "fail" }, "07_w4a16_gemm": { "run_id": "20260813_073002_grok_grok-4.6_07_w4a16_gemm", "correct": true, "has_solution": true, "peak_fraction": 0.2343, "elapsed_seconds": 4603, "annotation_verdict": "clean" } }, "pass_count": 5, "total_runs": 6 }, { "label": "kinetic-claude/kinetic-0715", "harness": "kinetic-claude", "model": "kinetic-0715", "effort": "", "results": { "01_fp8_gemm": { "run_id": "20260715_204204_kinetic-claude_kinetic-0715_01_fp8_gemm", "correct": true, "has_solution": true, "peak_fraction": 0.3196, "elapsed_seconds": null, "annotation_verdict": "clean", "output_tokens": 367883 }, "02_kda_cutlass": { "run_id": "20260716_091211_kinetic-claude_kinetic-0715_02_kda_cutlass", "correct": true, "has_solution": true, "peak_fraction": 0.0315, "elapsed_seconds": 7520, "annotation_verdict": "clean", "output_tokens": 768729 }, "03_paged_attention": { "run_id": "20260716_112603_kinetic-claude_kinetic-0715_03_paged_attention", "correct": true, "has_solution": true, "peak_fraction": 0.4855, "elapsed_seconds": 10955, "annotation_verdict": "clean", "output_tokens": 347311 }, "05_topk_bitonic": { "run_id": "20260716_112628_kinetic-claude_kinetic-0715_05_topk_bitonic", "correct": true, "has_solution": true, "peak_fraction": 0.064, "elapsed_seconds": 25263, "annotation_verdict": "clean", "output_tokens": 620587 }, "06_sonic_moe_swiglu": { "run_id": "20260715_204344_kinetic-claude_kinetic-0715_06_sonic_moe_swiglu", "correct": true, "has_solution": true, "peak_fraction": 0.0885, "elapsed_seconds": null, "annotation_verdict": "clean", "output_tokens": 314930 }, "07_w4a16_gemm": { "run_id": "20260716_112718_kinetic-claude_kinetic-0715_07_w4a16_gemm", "correct": false, "has_solution": true, "peak_fraction": null, "elapsed_seconds": 26139, "annotation_verdict": "clean", "output_tokens": 598166 } }, "pass_count": 5, "total_runs": 6 }, { "label": "longcat-claude/LongCat-2.0", "harness": "longcat-claude", "model": "LongCat-2.0", "effort": "", "results": { "01_fp8_gemm": { "run_id": "20260706_192143_longcat-claude_LongCat-2.0_01_fp8_gemm", "correct": true, "has_solution": true, "peak_fraction": 0.3286, "elapsed_seconds": 6169, "annotation_verdict": "clean", "output_tokens": 125326 }, "02_kda_cutlass": { "run_id": "20260707_131659_longcat-claude_LongCat-2.0_02_kda_cutlass", "correct": true, "has_solution": true, "peak_fraction": 0.0007, "elapsed_seconds": 8564, "annotation_verdict": "clean", "output_tokens": 162483 }, "03_paged_attention": { "run_id": "20260707_140259_longcat-claude_LongCat-2.0_03_paged_attention", "correct": true, "has_solution": true, "peak_fraction": 0.3192, "elapsed_seconds": 11854, "annotation_verdict": "clean", "output_tokens": 206402 }, "05_topk_bitonic": { "run_id": "20260707_044245_longcat-claude_LongCat-2.0_05_topk_bitonic", "correct": false, "has_solution": true, "peak_fraction": null, "elapsed_seconds": 13353, "annotation_verdict": "fail", "output_tokens": 85898 }, "06_sonic_moe_swiglu": { "run_id": "20260707_082519_longcat-claude_LongCat-2.0_06_sonic_moe_swiglu", "correct": true, "has_solution": true, "peak_fraction": 0.0711, "elapsed_seconds": 16203, "annotation_verdict": "clean", "output_tokens": 178900 }, "07_w4a16_gemm": { "run_id": "20260707_044245_longcat-claude_LongCat-2.0_07_w4a16_gemm", "correct": true, "has_solution": true, "peak_fraction": 0.1008, "elapsed_seconds": 33599, "annotation_verdict": "clean", "output_tokens": 311681 } }, "pass_count": 5, "total_runs": 6 }, { "label": "or-fable/anthropic/claude-fable-5", "harness": "or-fable", "model": "anthropic/claude-fable-5", "effort": "", "results": { "01_fp8_gemm": { "run_id": "20260721_152955_or-fable_anthropic_claude-fable-5_01_fp8_gemm", "correct": true, "has_solution": true, "peak_fraction": 0.3911, "elapsed_seconds": 3909, "annotation_verdict": "clean" }, "02_kda_cutlass": { "run_id": "20260719_031414_or-fable_anthropic_claude-fable-5_02_kda_cutlass", "correct": false, "has_solution": true, "peak_fraction": null, "elapsed_seconds": 4270, "annotation_verdict": "fail" }, "03_paged_attention": { "run_id": "20260719_040355_or-fable_anthropic_claude-fable-5_03_paged_attention", "correct": true, "has_solution": true, "peak_fraction": 0.4303, "elapsed_seconds": 10926, "annotation_verdict": "clean" }, "05_topk_bitonic": { "run_id": "20260719_042539_or-fable_anthropic_claude-fable-5_05_topk_bitonic", "correct": true, "has_solution": true, "peak_fraction": 0.0492, "elapsed_seconds": 7953, "annotation_verdict": "clean" }, "06_sonic_moe_swiglu": { "run_id": "20260719_063816_or-fable_anthropic_claude-fable-5_06_sonic_moe_swiglu", "correct": true, "has_solution": true, "peak_fraction": 0.1052, "elapsed_seconds": 5707, "annotation_verdict": "clean" }, "07_w4a16_gemm": { "run_id": "20260719_070614_or-fable_anthropic_claude-fable-5_07_w4a16_gemm", "correct": true, "has_solution": true, "peak_fraction": 0.2608, "elapsed_seconds": 5038, "annotation_verdict": "clean" } }, "pass_count": 5, "total_runs": 6 }, { "label": "zai-claude/glm-5.2", "harness": "zai-claude", "model": "glm-5.2", "effort": "", "results": { "01_fp8_gemm": { "run_id": "20260614_145529_zai-claude_glm-5.2_01_fp8_gemm", "correct": true, "has_solution": true, "peak_fraction": 0.4059, "elapsed_seconds": 5758, "annotation_verdict": "clean", "output_tokens": 149190 }, "02_kda_cutlass": { "run_id": "20260613_054111_zai-claude_glm-5.2_02_kda_cutlass", "correct": true, "has_solution": true, "peak_fraction": 0.03232, "elapsed_seconds": 14474, "annotation_verdict": "clean", "output_tokens": 197320 }, "03_paged_attention": { "run_id": "20260613_055815_zai-claude_glm-5.2_03_paged_attention", "correct": true, "has_solution": true, "peak_fraction": 0.6771, "elapsed_seconds": 14031, "annotation_verdict": "interesting", "output_tokens": 200088 }, "05_topk_bitonic": { "run_id": "20260613_060814_zai-claude_glm-5.2_05_topk_bitonic", "correct": true, "has_solution": true, "peak_fraction": 0.0341, "elapsed_seconds": 14137, "annotation_verdict": "interesting", "output_tokens": 218699 }, "06_sonic_moe_swiglu": { "run_id": "20260613_061433_zai-claude_glm-5.2_06_sonic_moe_swiglu", "correct": false, "has_solution": true, "peak_fraction": null, "elapsed_seconds": 14466, "annotation_verdict": "fail", "output_tokens": 155760 }, "07_w4a16_gemm": { "run_id": "20260613_061443_zai-claude_glm-5.2_07_w4a16_gemm", "correct": true, "has_solution": true, "peak_fraction": 0.3207, "elapsed_seconds": 16795, "annotation_verdict": "clean", "output_tokens": 196049 } }, "pass_count": 5, "total_runs": 6 }, { "label": "claude/claude-opus-4-8", "harness": "claude", "model": "claude-opus-4-8", "effort": "", "results": { "01_fp8_gemm": { "run_id": "20260614_144216_claude_claude-opus-4-8_01_fp8_gemm", "correct": true, "has_solution": true, "peak_fraction": 0.3855, "elapsed_seconds": 3006, "annotation_verdict": "clean", "output_tokens": 158464 }, "02_kda_cutlass": { "run_id": "20260613_042257_claude_claude-opus-4-8_02_kda_cutlass", "correct": false, "has_solution": true, "peak_fraction": null, "elapsed_seconds": 6827, "annotation_verdict": "fail", "output_tokens": 229902 }, "03_paged_attention": { "run_id": "20260613_042305_claude_claude-opus-4-8_03_paged_attention", "correct": true, "has_solution": true, "peak_fraction": 0.6706, "elapsed_seconds": 6697, "annotation_verdict": "interesting", "output_tokens": 145246 }, "05_topk_bitonic": { "run_id": "20260613_042313_claude_claude-opus-4-8_05_topk_bitonic", "correct": true, "has_solution": true, "peak_fraction": 0.0335, "elapsed_seconds": 8225, "annotation_verdict": "clean", "output_tokens": 295298 }, "06_sonic_moe_swiglu": { "run_id": "20260613_042321_claude_claude-opus-4-8_06_sonic_moe_swiglu", "correct": false, "has_solution": true, "peak_fraction": null, "elapsed_seconds": 6293, "annotation_verdict": "fail", "output_tokens": 124015 }, "07_w4a16_gemm": { "run_id": "20260613_115410_claude_claude-opus-4-8_07_w4a16_gemm", "correct": true, "has_solution": true, "peak_fraction": 0.2355, "elapsed_seconds": 13081, "annotation_verdict": "clean", "output_tokens": 355691 } }, "pass_count": 4, "total_runs": 6 }, { "label": "grok/grok-4.5 [max]", "harness": "grok", "model": "grok-4.5", "effort": "max", "results": { "01_fp8_gemm": { "run_id": "20260708_134136_grok_grok-4.5_01_fp8_gemm", "correct": true, "has_solution": true, "peak_fraction": 0.3367, "elapsed_seconds": 1397, "annotation_verdict": "clean" }, "02_kda_cutlass": { "run_id": "20260708_140941_grok_grok-4.5_02_kda_cutlass", "correct": true, "has_solution": true, "peak_fraction": 0.0198, "elapsed_seconds": 1783, "annotation_verdict": "clean" }, "03_paged_attention": { "run_id": "20260708_140941_grok_grok-4.5_03_paged_attention", "correct": true, "has_solution": true, "peak_fraction": 0.6538, "elapsed_seconds": 1443, "annotation_verdict": "clean" }, "05_topk_bitonic": { "run_id": "20260708_143924_grok_grok-4.5_05_topk_bitonic", "correct": true, "has_solution": true, "peak_fraction": 0.0293, "elapsed_seconds": 3271, "annotation_verdict": "clean" }, "06_sonic_moe_swiglu": { "run_id": "20260708_143344_grok_grok-4.5_06_sonic_moe_swiglu", "correct": false, "has_solution": true, "peak_fraction": null, "elapsed_seconds": 2748, "annotation_verdict": "fail" }, "07_w4a16_gemm": { "run_id": "20260708_153355_grok_grok-4.5_07_w4a16_gemm", "correct": false, "has_solution": true, "peak_fraction": null, "elapsed_seconds": 1583, "annotation_verdict": "clean" } }, "pass_count": 4, "total_runs": 6 }, { "label": "minimax-claude/MiniMax-M3", "harness": "minimax-claude", "model": "MiniMax-M3", "effort": "", "results": { "01_fp8_gemm": { "run_id": "20260614_153222_minimax-claude_MiniMax-M3_01_fp8_gemm", "correct": true, "has_solution": true, "peak_fraction": 0.3661, "elapsed_seconds": 10146, "annotation_verdict": "clean", "output_tokens": 161138 }, "02_kda_cutlass": { "run_id": "20260613_064019_minimax-claude_MiniMax-M3_02_kda_cutlass", "correct": false, "has_solution": true, "peak_fraction": null, "elapsed_seconds": 21849, "annotation_verdict": "bug", "output_tokens": 16144 }, "03_paged_attention": { "run_id": "20260613_081306_minimax-claude_MiniMax-M3_03_paged_attention", "correct": true, "has_solution": true, "peak_fraction": 0.5129, "elapsed_seconds": 21282, "annotation_verdict": "clean", "output_tokens": 201109 }, "05_topk_bitonic": { "run_id": "20260613_081745_minimax-claude_MiniMax-M3_05_topk_bitonic", "correct": true, "has_solution": true, "peak_fraction": 0.0057, "elapsed_seconds": 21695, "annotation_verdict": "interesting", "output_tokens": 193477 }, "06_sonic_moe_swiglu": { "run_id": "20260613_094226_minimax-claude_MiniMax-M3_06_sonic_moe_swiglu", "correct": false, "has_solution": true, "peak_fraction": null, "elapsed_seconds": 21817, "annotation_verdict": "fail", "output_tokens": 136204 }, "07_w4a16_gemm": { "run_id": "20260613_095207_minimax-claude_MiniMax-M3_07_w4a16_gemm", "correct": true, "has_solution": true, "peak_fraction": 0.145, "elapsed_seconds": 16805, "annotation_verdict": "clean", "output_tokens": 150975 } }, "pass_count": 4, "total_runs": 6 }, { "label": "or-fable/deepseek/deepseek-v4-flash-0731", "harness": "or-fable", "model": "deepseek/deepseek-v4-flash-0731", "effort": "", "results": { "01_fp8_gemm": { "run_id": "20260801_205612_or-fable_deepseek_deepseek-v4-flash-0731_01_fp8_gemm", "correct": true, "has_solution": true, "peak_fraction": 0.409, "elapsed_seconds": 2751, "annotation_verdict": "clean" }, "02_kda_cutlass": { "run_id": "20260802_045225_or-fable_deepseek_deepseek-v4-flash-0731_02_kda_cutlass", "correct": false, "has_solution": true, "peak_fraction": null, "elapsed_seconds": 4975, "annotation_verdict": "fail" }, "03_paged_attention": { "run_id": "20260801_214207_or-fable_deepseek_deepseek-v4-flash-0731_03_paged_attention", "correct": true, "has_solution": true, "peak_fraction": 0.486, "elapsed_seconds": 6886, "annotation_verdict": "clean" }, "05_topk_bitonic": { "run_id": "20260801_225649_or-fable_deepseek_deepseek-v4-flash-0731_05_topk_bitonic", "correct": true, "has_solution": true, "peak_fraction": 0.0285, "elapsed_seconds": 8385, "annotation_verdict": "clean" }, "06_sonic_moe_swiglu": { "run_id": "20260801_233657_or-fable_deepseek_deepseek-v4-flash-0731_06_sonic_moe_swiglu", "correct": false, "has_solution": true, "peak_fraction": null, "elapsed_seconds": 5331, "annotation_verdict": "fail" }, "07_w4a16_gemm": { "run_id": "20260802_011638_or-fable_deepseek_deepseek-v4-flash-0731_07_w4a16_gemm", "correct": true, "has_solution": true, "peak_fraction": 0.1562, "elapsed_seconds": 11337, "annotation_verdict": "clean" } }, "pass_count": 4, "total_runs": 6 }, { "label": "codex/gpt-5.6-sol [xhigh]", "harness": "codex", "model": "gpt-5.6-sol", "effort": "xhigh", "results": { "01_fp8_gemm": { "run_id": "20260709_172635_codex_gpt-5.6-sol_01_fp8_gemm", "correct": true, "has_solution": true, "peak_fraction": 0.3871, "elapsed_seconds": 1264, "annotation_verdict": "clean", "output_tokens": 41421 }, "02_kda_cutlass": { "run_id": "20260709_192708_codex_gpt-5.6-sol_02_kda_cutlass", "correct": false, "has_solution": true, "peak_fraction": null, "elapsed_seconds": 1803, "annotation_verdict": "fail", "output_tokens": 53060 }, "03_paged_attention": { "run_id": "20260709_174740_codex_gpt-5.6-sol_03_paged_attention", "correct": false, "has_solution": true, "peak_fraction": null, "elapsed_seconds": 1568, "annotation_verdict": "fail", "output_tokens": 38793 }, "05_topk_bitonic": { "run_id": "20260718_215148_codex_gpt-5.6-sol_05_topk_bitonic", "correct": true, "has_solution": true, "peak_fraction": 0.0412, "elapsed_seconds": 2110, "annotation_verdict": "clean", "output_tokens": 60126 }, "06_sonic_moe_swiglu": { "run_id": "20260718_222659_codex_gpt-5.6-sol_06_sonic_moe_swiglu", "correct": false, "has_solution": true, "peak_fraction": null, "elapsed_seconds": 3761, "annotation_verdict": "fail", "output_tokens": 74793 }, "07_w4a16_gemm": { "run_id": "20260709_184850_codex_gpt-5.6-sol_07_w4a16_gemm", "correct": true, "has_solution": true, "peak_fraction": 0.198, "elapsed_seconds": 2265, "annotation_verdict": "clean", "output_tokens": 57998 } }, "pass_count": 3, "total_runs": 6 }, { "label": "or-fable/qwen/qwen3.8-max [xhigh]", "harness": "or-fable", "model": "qwen/qwen3.8-max", "effort": "xhigh", "results": { "02_kda_cutlass": { "run_id": "20260803_214319_or-fable_qwen_qwen3.8-max_02_kda_cutlass", "correct": false, "has_solution": true, "peak_fraction": null, "elapsed_seconds": 5512, "annotation_verdict": "fail" }, "03_paged_attention": { "run_id": "20260805_021249_or-fable_qwen_qwen3.8-max_03_paged_attention", "correct": true, "has_solution": true, "peak_fraction": 0.6926, "elapsed_seconds": 9630, "annotation_verdict": "clean", "output_tokens": 223326 }, "05_topk_bitonic": { "run_id": "20260804_025949_or-fable_qwen_qwen3.8-max_05_topk_bitonic", "correct": true, "has_solution": true, "peak_fraction": 0.0397, "elapsed_seconds": 10813, "annotation_verdict": "clean" }, "06_sonic_moe_swiglu": { "run_id": "20260803_231512_or-fable_qwen_qwen3.8-max_06_sonic_moe_swiglu", "correct": true, "has_solution": true, "peak_fraction": 0.1067, "elapsed_seconds": 11797, "annotation_verdict": "clean", "output_tokens": 175966 }, "07_w4a16_gemm": { "run_id": "20260804_025959_or-fable_qwen_qwen3.8-max_07_w4a16_gemm", "correct": true, "has_solution": true, "peak_fraction": 0.1457, "elapsed_seconds": 4152, "annotation_verdict": "reward_hack", "invalid_reason": "reward_hack" } }, "pass_count": 3, "total_runs": 5 }, { "label": "zai-claude/glm-5.3", "harness": "zai-claude", "model": "glm-5.3", "effort": "", "results": { "02_kda_cutlass": { "run_id": "20260822_053629_zai-claude_glm-5.3_02_kda_cutlass", "correct": false, "has_solution": true, "peak_fraction": null, "elapsed_seconds": 17956, "annotation_verdict": "fail", "output_tokens": 583747 }, "03_paged_attention": { "run_id": "20260822_053649_zai-claude_glm-5.3_03_paged_attention", "correct": true, "has_solution": true, "peak_fraction": 0.6319, "elapsed_seconds": 11725, "annotation_verdict": "clean", "output_tokens": 501633 }, "05_topk_bitonic": { "run_id": "20260822_053709_zai-claude_glm-5.3_05_topk_bitonic", "correct": true, "has_solution": true, "peak_fraction": 0.0885, "elapsed_seconds": 8165, "annotation_verdict": "clean", "output_tokens": 317280 }, "06_sonic_moe_swiglu": { "run_id": "20260822_075101_zai-claude_glm-5.3_06_sonic_moe_swiglu", "correct": false, "has_solution": true, "peak_fraction": null, "elapsed_seconds": 9882, "annotation_verdict": "fail", "output_tokens": 265744 }, "07_w4a16_gemm": { "run_id": "20260822_075321_zai-claude_glm-5.3_07_w4a16_gemm", "correct": true, "has_solution": true, "peak_fraction": 0.3653, "elapsed_seconds": 6155, "annotation_verdict": "clean", "output_tokens": 202809 } }, "pass_count": 3, "total_runs": 5 }, { "label": "or-fable/stealth/ox-alpha", "harness": "or-fable", "model": "stealth/ox-alpha", "effort": "", "results": { "02_kda_cutlass": { "run_id": "20260822_061013_or-fable_stealth_ox-alpha_02_kda_cutlass", "correct": false, "has_solution": true, "peak_fraction": null, "elapsed_seconds": 7644, "annotation_verdict": "fail" }, "05_topk_bitonic": { "run_id": "20260822_061053_or-fable_stealth_ox-alpha_05_topk_bitonic", "correct": true, "has_solution": true, "peak_fraction": 0.0717, "elapsed_seconds": 12048, "annotation_verdict": "clean" }, "06_sonic_moe_swiglu": { "run_id": "20260822_061328_or-fable_stealth_ox-alpha_06_sonic_moe_swiglu", "correct": true, "has_solution": true, "peak_fraction": 0.1021, "elapsed_seconds": 22283, "annotation_verdict": "clean" } }, "pass_count": 2, "total_runs": 3 }, { "label": "or-opus/anthropic/claude-opus-5 [max]", "harness": "or-opus", "model": "anthropic/claude-opus-5", "effort": "max", "results": { "05_topk_bitonic": { "run_id": "20260725_002808_or-opus_anthropic_claude-opus-5_05_topk_bitonic", "correct": true, "has_solution": true, "peak_fraction": 0.0946, "elapsed_seconds": 25426, "annotation_verdict": "clean" }, "07_w4a16_gemm": { "run_id": "20260725_002828_or-opus_anthropic_claude-opus-5_07_w4a16_gemm", "correct": true, "has_solution": true, "peak_fraction": 0.3708, "elapsed_seconds": 15840, "annotation_verdict": "clean" } }, "pass_count": 2, "total_runs": 2 }, { "label": "claude/claude-fable-5 [max]", "harness": "claude", "model": "claude-fable-5", "effort": "max", "results": { "01_fp8_gemm": { "run_id": "20260716_090642_claude_claude-fable-5_01_fp8_gemm", "correct": true, "has_solution": true, "peak_fraction": 0.348, "elapsed_seconds": 3560, "annotation_verdict": "clean", "output_tokens": 229050 } }, "pass_count": 1, "total_runs": 1 }, { "label": "claude/claude-opus-4-8 [max]", "harness": "claude", "model": "claude-opus-4-8", "effort": "max", "results": { "07_w4a16_gemm": { "run_id": "20260719_030522_claude_claude-opus-4-8_07_w4a16_gemm", "correct": true, "has_solution": true, "peak_fraction": 0.1143, "elapsed_seconds": 1630, "annotation_verdict": "clean", "output_tokens": 355691 } }, "pass_count": 1, "total_runs": 1 }, { "label": "qwen-claude/qwen3.8-max", "harness": "qwen-claude", "model": "qwen3.8-max", "effort": "", "results": { "01_fp8_gemm": { "run_id": "20260803_034401_qwen-claude_qwen3.8-max_01_fp8_gemm", "correct": true, "has_solution": true, "peak_fraction": 0.3269, "elapsed_seconds": 6443, "annotation_verdict": "clean", "output_tokens": 237300 } }, "pass_count": 1, "total_runs": 1 }, { "label": "grok/grok-4.5 [high]", "harness": "grok", "model": "grok-4.5", "effort": "high", "results": { "07_w4a16_gemm": { "run_id": "20260719_030522_grok_grok-4.5_07_w4a16_gemm", "correct": false, "has_solution": true, "peak_fraction": null, "elapsed_seconds": 3349 } }, "pass_count": 0, "total_runs": 1 } ], "per_problem": { "01_fp8_gemm": { "n_attempted": 16, "n_passed": 16, "best_peak_fraction": 0.409, "best_model": "or-fable/deepseek/deepseek-v4-flash-0731", "ranked_passes": [ { "model": "or-fable/deepseek/deepseek-v4-flash-0731", "peak_fraction": 0.409 }, { "model": "zai-claude/glm-5.2", "peak_fraction": 0.4059 }, { "model": "deepseek-claude/deepseek-v4-pro", "peak_fraction": 0.3959 }, { "model": "or-fable/anthropic/claude-fable-5", "peak_fraction": 0.3911 }, { "model": "codex/gpt-5.6-sol [xhigh]", "peak_fraction": 0.3871 }, { "model": "claude/claude-opus-4-8", "peak_fraction": 0.3855 }, { "model": "grok/grok-4.6 [xhigh]", "peak_fraction": 0.375 }, { "model": "minimax-claude/MiniMax-M3", "peak_fraction": 0.3661 }, { "model": "kinetic-claude/kinetic-0715[1m]", "peak_fraction": 0.3529 }, { "model": "claude/claude-fable-5 [max]", "peak_fraction": 0.348 }, { "model": "grok/grok-4.5 [max]", "peak_fraction": 0.3367 }, { "model": "deepseek-claude/deepseek-flash", "peak_fraction": 0.331 }, { "model": "longcat-claude/LongCat-2.0", "peak_fraction": 0.3286 }, { "model": "qwen-claude/qwen3.8-max", "peak_fraction": 0.3269 }, { "model": "or-fable/deepseek/deepseek-v4.1-flash-1017", "peak_fraction": 0.3242 }, { "model": "kinetic-claude/kinetic-0715", "peak_fraction": 0.3196 } ] }, "02_kda_cutlass": { "n_attempted": 17, "n_passed": 9, "best_peak_fraction": 0.0513, "best_model": "deepseek-claude/deepseek-v4-pro", "ranked_passes": [ { "model": "deepseek-claude/deepseek-v4-pro", "peak_fraction": 0.0513 }, { "model": "kinetic-claude/kinetic-0715[1m]", "peak_fraction": 0.0493 }, { "model": "grok/grok-4.6 [xhigh]", "peak_fraction": 0.0406 }, { "model": "or-fable/deepseek/deepseek-v4.1-flash-1017", "peak_fraction": 0.0403 }, { "model": "deepseek-claude/deepseek-flash", "peak_fraction": 0.0374 }, { "model": "zai-claude/glm-5.2", "peak_fraction": 0.03232 }, { "model": "kinetic-claude/kinetic-0715", "peak_fraction": 0.0315 }, { "model": "grok/grok-4.5 [max]", "peak_fraction": 0.0198 }, { "model": "longcat-claude/LongCat-2.0", "peak_fraction": 0.0007 } ] }, "03_paged_attention": { "n_attempted": 16, "n_passed": 15, "best_peak_fraction": 0.6926, "best_model": "or-fable/qwen/qwen3.8-max [xhigh]", "ranked_passes": [ { "model": "or-fable/qwen/qwen3.8-max [xhigh]", "peak_fraction": 0.6926 }, { "model": "zai-claude/glm-5.2", "peak_fraction": 0.6771 }, { "model": "claude/claude-opus-4-8", "peak_fraction": 0.6706 }, { "model": "grok/grok-4.5 [max]", "peak_fraction": 0.6538 }, { "model": "zai-claude/glm-5.3", "peak_fraction": 0.6319 }, { "model": "grok/grok-4.6 [xhigh]", "peak_fraction": 0.6217 }, { "model": "kinetic-claude/kinetic-0715[1m]", "peak_fraction": 0.5811 }, { "model": "minimax-claude/MiniMax-M3", "peak_fraction": 0.5129 }, { "model": "deepseek-claude/deepseek-v4-pro", "peak_fraction": 0.5 }, { "model": "or-fable/deepseek/deepseek-v4-flash-0731", "peak_fraction": 0.486 }, { "model": "kinetic-claude/kinetic-0715", "peak_fraction": 0.4855 }, { "model": "or-fable/deepseek/deepseek-v4.1-flash-1017", "peak_fraction": 0.4586 }, { "model": "or-fable/anthropic/claude-fable-5", "peak_fraction": 0.4303 }, { "model": "deepseek-claude/deepseek-flash", "peak_fraction": 0.3622 }, { "model": "longcat-claude/LongCat-2.0", "peak_fraction": 0.3192 } ] }, "05_topk_bitonic": { "n_attempted": 18, "n_passed": 17, "best_peak_fraction": 0.0946, "best_model": "or-opus/anthropic/claude-opus-5 [max]", "ranked_passes": [ { "model": "or-opus/anthropic/claude-opus-5 [max]", "peak_fraction": 0.0946 }, { "model": "deepseek-claude/deepseek-v4-pro", "peak_fraction": 0.0935 }, { "model": "or-fable/deepseek/deepseek-v4.1-flash-1017", "peak_fraction": 0.0897 }, { "model": "kinetic-claude/kinetic-0715[1m]", "peak_fraction": 0.0895 }, { "model": "zai-claude/glm-5.3", "peak_fraction": 0.0885 }, { "model": "or-fable/stealth/ox-alpha", "peak_fraction": 0.0717 }, { "model": "kinetic-claude/kinetic-0715", "peak_fraction": 0.064 }, { "model": "or-fable/anthropic/claude-fable-5", "peak_fraction": 0.0492 }, { "model": "deepseek-claude/deepseek-flash", "peak_fraction": 0.0425 }, { "model": "codex/gpt-5.6-sol [xhigh]", "peak_fraction": 0.0412 }, { "model": "or-fable/qwen/qwen3.8-max [xhigh]", "peak_fraction": 0.0397 }, { "model": "grok/grok-4.6 [xhigh]", "peak_fraction": 0.0368 }, { "model": "zai-claude/glm-5.2", "peak_fraction": 0.0341 }, { "model": "claude/claude-opus-4-8", "peak_fraction": 0.0335 }, { "model": "grok/grok-4.5 [max]", "peak_fraction": 0.0293 }, { "model": "or-fable/deepseek/deepseek-v4-flash-0731", "peak_fraction": 0.0285 }, { "model": "minimax-claude/MiniMax-M3", "peak_fraction": 0.0057 } ] }, "06_sonic_moe_swiglu": { "n_attempted": 17, "n_passed": 9, "best_peak_fraction": 0.1081, "best_model": "deepseek-claude/deepseek-flash", "ranked_passes": [ { "model": "deepseek-claude/deepseek-flash", "peak_fraction": 0.1081 }, { "model": "or-fable/qwen/qwen3.8-max [xhigh]", "peak_fraction": 0.1067 }, { "model": "or-fable/deepseek/deepseek-v4.1-flash-1017", "peak_fraction": 0.106 }, { "model": "or-fable/anthropic/claude-fable-5", "peak_fraction": 0.1052 }, { "model": "or-fable/stealth/ox-alpha", "peak_fraction": 0.1021 }, { "model": "kinetic-claude/kinetic-0715", "peak_fraction": 0.0885 }, { "model": "longcat-claude/LongCat-2.0", "peak_fraction": 0.0711 }, { "model": "deepseek-claude/deepseek-v4-pro", "peak_fraction": 0.0533 }, { "model": "kinetic-claude/kinetic-0715[1m]", "peak_fraction": 0.0329 } ] }, "07_w4a16_gemm": { "n_attempted": 19, "n_passed": 15, "best_peak_fraction": 0.3708, "best_model": "or-opus/anthropic/claude-opus-5 [max]", "ranked_passes": [ { "model": "or-opus/anthropic/claude-opus-5 [max]", "peak_fraction": 0.3708 }, { "model": "zai-claude/glm-5.3", "peak_fraction": 0.3653 }, { "model": "zai-claude/glm-5.2", "peak_fraction": 0.3207 }, { "model": "deepseek-claude/deepseek-v4-pro", "peak_fraction": 0.3103 }, { "model": "or-fable/anthropic/claude-fable-5", "peak_fraction": 0.2608 }, { "model": "or-fable/deepseek/deepseek-v4.1-flash-1017", "peak_fraction": 0.2513 }, { "model": "deepseek-claude/deepseek-flash", "peak_fraction": 0.2491 }, { "model": "claude/claude-opus-4-8", "peak_fraction": 0.2355 }, { "model": "grok/grok-4.6 [xhigh]", "peak_fraction": 0.2343 }, { "model": "codex/gpt-5.6-sol [xhigh]", "peak_fraction": 0.198 }, { "model": "or-fable/deepseek/deepseek-v4-flash-0731", "peak_fraction": 0.1562 }, { "model": "minimax-claude/MiniMax-M3", "peak_fraction": 0.145 }, { "model": "claude/claude-opus-4-8 [max]", "peak_fraction": 0.1143 }, { "model": "longcat-claude/LongCat-2.0", "peak_fraction": 0.1008 }, { "model": "kinetic-claude/kinetic-0715[1m]", "peak_fraction": 0.027 } ] } }, "generated_from_summary": { "input": "benchmarks/hard/outputs/runs", "tag": "v2", "imported_rows": 22 } }