{ "format": "roomote-role-evals/v1", "publishedAt": "2026-08-19T23:04:55.906Z", "evals": [ { "id": "coding", "name": "Coding — repository implementation", "description": "Runs Roomote's production coding harness repeatedly on a compact set of real repository bugs, pins helper roles to the balanced baseline while the candidate model reviews and advises itself, and requires the work to ship as a pull request — undelivered attempts score zero, with raw implementation quality reported separately as a diagnostic.", "role": "coding", "cases": 6, "qualityMethod": "task-macro delivered acceptance", "updatedAt": "2026-08-07T21:01:35.987Z", "results": [ { "model": "openai/gpt-5.6-luna@medium", "qualityScore": 96, "latencyP50Ms": 93599, "latencyP95Ms": 256386, "totalDurationMs": 780257, "costUsd": 0.21143278, "costPerCaseUsd": 0.03523879666666666, "tokens": 31851, "casesAttempted": 6, "metrics": [ { "label": "task acceptance", "value": "96%", "hint": "Task-macro acceptance of delivered work; attempts that never ship a PR score zero — higher is better", "sort": 0.9583333333333334 }, { "label": "implementation acceptance", "value": "96%", "hint": "Task-macro acceptance of the produced diff regardless of delivery — diagnostic for separating coding quality from delivery reliability", "sort": 0.9583333333333334 }, { "label": "attempt score range", "value": "75–100%", "hint": "Observed delivered-acceptance range across repeated task attempts", "sort": null }, { "label": "tasks completed", "value": "5/6 (83%)", "hint": "Tasks with every acceptance criterion met and a delivered PR — higher is better", "sort": 0.8333333333333334 }, { "label": "PR delivery", "value": "6/6 (100%)", "hint": "Tasks where the coding agent completed Roomote's delivery flow — higher is better", "sort": 1 }, { "label": "judge agreement", "value": "99%", "hint": "Average majority agreement across acceptance criteria — higher means the grading panel was more consistent", "sort": 0.986111111111111 }, { "label": "valid attempts", "value": "6/6", "hint": "Infrastructure-valid attempts completed out of the frozen benchmark requirement", "sort": 1 }, { "label": "gate overrides", "value": "0/18 criteria", "hint": "Criteria the judge majority passed but a deterministic objective gate forced to fail — review these disagreements before trusting close rankings", "sort": null } ] }, { "model": "anthropic/claude-sonnet-5@medium", "qualityScore": 96, "latencyP50Ms": 122392, "latencyP95Ms": 391014, "totalDurationMs": 1299955, "costUsd": 4.799479799999999, "costPerCaseUsd": 0.7999132999999999, "tokens": 71454, "casesAttempted": 6, "metrics": [ { "label": "task acceptance", "value": "96%", "hint": "Task-macro acceptance of delivered work; attempts that never ship a PR score zero — higher is better", "sort": 0.9583333333333334 }, { "label": "implementation acceptance", "value": "96%", "hint": "Task-macro acceptance of the produced diff regardless of delivery — diagnostic for separating coding quality from delivery reliability", "sort": 0.9583333333333334 }, { "label": "attempt score range", "value": "75–100%", "hint": "Observed delivered-acceptance range across repeated task attempts", "sort": null }, { "label": "tasks completed", "value": "5/6 (83%)", "hint": "Tasks with every acceptance criterion met and a delivered PR — higher is better", "sort": 0.8333333333333334 }, { "label": "PR delivery", "value": "6/6 (100%)", "hint": "Tasks where the coding agent completed Roomote's delivery flow — higher is better", "sort": 1 }, { "label": "judge agreement", "value": "98%", "hint": "Average majority agreement across acceptance criteria — higher means the grading panel was more consistent", "sort": 0.9814814814814815 }, { "label": "valid attempts", "value": "6/6", "hint": "Infrastructure-valid attempts completed out of the frozen benchmark requirement", "sort": 1 }, { "label": "gate overrides", "value": "0/18 criteria", "hint": "Criteria the judge majority passed but a deterministic objective gate forced to fail — review these disagreements before trusting close rankings", "sort": null } ] }, { "model": "x-ai/grok-4.5@medium", "qualityScore": 96, "latencyP50Ms": 141564, "latencyP95Ms": 271775, "totalDurationMs": 997438, "costUsd": 3.2827336000000003, "costPerCaseUsd": 0.5471222666666667, "tokens": 568492, "casesAttempted": 6, "metrics": [ { "label": "task acceptance", "value": "96%", "hint": "Task-macro acceptance of delivered work; attempts that never ship a PR score zero — higher is better", "sort": 0.9583333333333334 }, { "label": "implementation acceptance", "value": "96%", "hint": "Task-macro acceptance of the produced diff regardless of delivery — diagnostic for separating coding quality from delivery reliability", "sort": 0.9583333333333334 }, { "label": "attempt score range", "value": "75–100%", "hint": "Observed delivered-acceptance range across repeated task attempts", "sort": null }, { "label": "tasks completed", "value": "5/6 (83%)", "hint": "Tasks with every acceptance criterion met and a delivered PR — higher is better", "sort": 0.8333333333333334 }, { "label": "PR delivery", "value": "6/6 (100%)", "hint": "Tasks where the coding agent completed Roomote's delivery flow — higher is better", "sort": 1 }, { "label": "judge agreement", "value": "98%", "hint": "Average majority agreement across acceptance criteria — higher means the grading panel was more consistent", "sort": 0.9814814814814815 }, { "label": "valid attempts", "value": "6/6", "hint": "Infrastructure-valid attempts completed out of the frozen benchmark requirement", "sort": 1 }, { "label": "gate overrides", "value": "0/18 criteria", "hint": "Criteria the judge majority passed but a deterministic objective gate forced to fail — review these disagreements before trusting close rankings", "sort": null } ] }, { "model": "anthropic/claude-opus-5@medium", "qualityScore": 96, "latencyP50Ms": 168731, "latencyP95Ms": 717505, "totalDurationMs": 1907240, "costUsd": 16.451258250000002, "costPerCaseUsd": 2.7418763750000004, "tokens": 103189, "casesAttempted": 6, "metrics": [ { "label": "task acceptance", "value": "96%", "hint": "Task-macro acceptance of delivered work; attempts that never ship a PR score zero — higher is better", "sort": 0.9583333333333334 }, { "label": "implementation acceptance", "value": "96%", "hint": "Task-macro acceptance of the produced diff regardless of delivery — diagnostic for separating coding quality from delivery reliability", "sort": 0.9583333333333334 }, { "label": "attempt score range", "value": "75–100%", "hint": "Observed delivered-acceptance range across repeated task attempts", "sort": null }, { "label": "tasks completed", "value": "5/6 (83%)", "hint": "Tasks with every acceptance criterion met and a delivered PR — higher is better", "sort": 0.8333333333333334 }, { "label": "PR delivery", "value": "6/6 (100%)", "hint": "Tasks where the coding agent completed Roomote's delivery flow — higher is better", "sort": 1 }, { "label": "judge agreement", "value": "100%", "hint": "Average majority agreement across acceptance criteria — higher means the grading panel was more consistent", "sort": 1 }, { "label": "valid attempts", "value": "6/6", "hint": "Infrastructure-valid attempts completed out of the frozen benchmark requirement", "sort": 1 }, { "label": "gate overrides", "value": "0/18 criteria", "hint": "Criteria the judge majority passed but a deterministic objective gate forced to fail — review these disagreements before trusting close rankings", "sort": null } ] }, { "model": "anthropic/claude-fable-5@medium", "qualityScore": 96, "latencyP50Ms": 175072, "latencyP95Ms": 435846, "totalDurationMs": 1431216, "costUsd": 23.9569355, "costPerCaseUsd": 3.9928225833333335, "tokens": 80241, "casesAttempted": 6, "metrics": [ { "label": "task acceptance", "value": "96%", "hint": "Task-macro acceptance of delivered work; attempts that never ship a PR score zero — higher is better", "sort": 0.9583333333333334 }, { "label": "implementation acceptance", "value": "96%", "hint": "Task-macro acceptance of the produced diff regardless of delivery — diagnostic for separating coding quality from delivery reliability", "sort": 0.9583333333333334 }, { "label": "attempt score range", "value": "75–100%", "hint": "Observed delivered-acceptance range across repeated task attempts", "sort": null }, { "label": "tasks completed", "value": "5/6 (83%)", "hint": "Tasks with every acceptance criterion met and a delivered PR — higher is better", "sort": 0.8333333333333334 }, { "label": "PR delivery", "value": "6/6 (100%)", "hint": "Tasks where the coding agent completed Roomote's delivery flow — higher is better", "sort": 1 }, { "label": "judge agreement", "value": "98%", "hint": "Average majority agreement across acceptance criteria — higher means the grading panel was more consistent", "sort": 0.9814814814814815 }, { "label": "valid attempts", "value": "6/6", "hint": "Infrastructure-valid attempts completed out of the frozen benchmark requirement", "sort": 1 }, { "label": "gate overrides", "value": "0/18 criteria", "hint": "Criteria the judge majority passed but a deterministic objective gate forced to fail — review these disagreements before trusting close rankings", "sort": null } ] }, { "model": "moonshotai/kimi-k3@medium", "qualityScore": 96, "latencyP50Ms": 203326, "latencyP95Ms": 444517, "totalDurationMs": 1529988, "costUsd": 5.360270699999999, "costPerCaseUsd": 0.8933784499999998, "tokens": 998024, "casesAttempted": 6, "metrics": [ { "label": "task acceptance", "value": "96%", "hint": "Task-macro acceptance of delivered work; attempts that never ship a PR score zero — higher is better", "sort": 0.9583333333333334 }, { "label": "implementation acceptance", "value": "96%", "hint": "Task-macro acceptance of the produced diff regardless of delivery — diagnostic for separating coding quality from delivery reliability", "sort": 0.9583333333333334 }, { "label": "attempt score range", "value": "75–100%", "hint": "Observed delivered-acceptance range across repeated task attempts", "sort": null }, { "label": "tasks completed", "value": "5/6 (83%)", "hint": "Tasks with every acceptance criterion met and a delivered PR — higher is better", "sort": 0.8333333333333334 }, { "label": "PR delivery", "value": "6/6 (100%)", "hint": "Tasks where the coding agent completed Roomote's delivery flow — higher is better", "sort": 1 }, { "label": "judge agreement", "value": "94%", "hint": "Average majority agreement across acceptance criteria — higher means the grading panel was more consistent", "sort": 0.9444444444444445 }, { "label": "valid attempts", "value": "6/6", "hint": "Infrastructure-valid attempts completed out of the frozen benchmark requirement", "sort": 1 }, { "label": "gate overrides", "value": "0/18 criteria", "hint": "Criteria the judge majority passed but a deterministic objective gate forced to fail — review these disagreements before trusting close rankings", "sort": null } ] }, { "model": "openai/gpt-5.6-terra@medium", "qualityScore": 90, "latencyP50Ms": 121405, "latencyP95Ms": 334537, "totalDurationMs": 1027784, "costUsd": 2.0704287, "costPerCaseUsd": 0.34507145, "tokens": 38342, "casesAttempted": 6, "metrics": [ { "label": "task acceptance", "value": "90%", "hint": "Task-macro acceptance of delivered work; attempts that never ship a PR score zero — higher is better", "sort": 0.9027777777777778 }, { "label": "implementation acceptance", "value": "90%", "hint": "Task-macro acceptance of the produced diff regardless of delivery — diagnostic for separating coding quality from delivery reliability", "sort": 0.9027777777777778 }, { "label": "attempt score range", "value": "67–100%", "hint": "Observed delivered-acceptance range across repeated task attempts", "sort": null }, { "label": "tasks completed", "value": "4/6 (67%)", "hint": "Tasks with every acceptance criterion met and a delivered PR — higher is better", "sort": 0.6666666666666666 }, { "label": "PR delivery", "value": "6/6 (100%)", "hint": "Tasks where the coding agent completed Roomote's delivery flow — higher is better", "sort": 1 }, { "label": "judge agreement", "value": "98%", "hint": "Average majority agreement across acceptance criteria — higher means the grading panel was more consistent", "sort": 0.9814814814814815 }, { "label": "valid attempts", "value": "6/6", "hint": "Infrastructure-valid attempts completed out of the frozen benchmark requirement", "sort": 1 }, { "label": "gate overrides", "value": "1/18 criteria", "hint": "Criteria the judge majority passed but a deterministic objective gate forced to fail — review these disagreements before trusting close rankings", "sort": null } ] }, { "model": "openai/gpt-5.6-sol@medium", "qualityScore": 90, "latencyP50Ms": 207463, "latencyP95Ms": 541935, "totalDurationMs": 1513067, "costUsd": 10.801269499999998, "costPerCaseUsd": 1.800211583333333, "tokens": 38247, "casesAttempted": 6, "metrics": [ { "label": "task acceptance", "value": "90%", "hint": "Task-macro acceptance of delivered work; attempts that never ship a PR score zero — higher is better", "sort": 0.9027777777777778 }, { "label": "implementation acceptance", "value": "90%", "hint": "Task-macro acceptance of the produced diff regardless of delivery — diagnostic for separating coding quality from delivery reliability", "sort": 0.9027777777777778 }, { "label": "attempt score range", "value": "67–100%", "hint": "Observed delivered-acceptance range across repeated task attempts", "sort": null }, { "label": "tasks completed", "value": "4/6 (67%)", "hint": "Tasks with every acceptance criterion met and a delivered PR — higher is better", "sort": 0.6666666666666666 }, { "label": "PR delivery", "value": "6/6 (100%)", "hint": "Tasks where the coding agent completed Roomote's delivery flow — higher is better", "sort": 1 }, { "label": "judge agreement", "value": "98%", "hint": "Average majority agreement across acceptance criteria — higher means the grading panel was more consistent", "sort": 0.9814814814814815 }, { "label": "valid attempts", "value": "6/6", "hint": "Infrastructure-valid attempts completed out of the frozen benchmark requirement", "sort": 1 }, { "label": "gate overrides", "value": "1/18 criteria", "hint": "Criteria the judge majority passed but a deterministic objective gate forced to fail — review these disagreements before trusting close rankings", "sort": null } ] }, { "model": "z-ai/glm-5.2@medium", "qualityScore": 90, "latencyP50Ms": 280536, "latencyP95Ms": 684401, "totalDurationMs": 2470175, "costUsd": 2.1933502729199996, "costPerCaseUsd": 0.3655583788199999, "tokens": 1982192, "casesAttempted": 6, "metrics": [ { "label": "task acceptance", "value": "90%", "hint": "Task-macro acceptance of delivered work; attempts that never ship a PR score zero — higher is better", "sort": 0.9027777777777778 }, { "label": "implementation acceptance", "value": "90%", "hint": "Task-macro acceptance of the produced diff regardless of delivery — diagnostic for separating coding quality from delivery reliability", "sort": 0.9027777777777778 }, { "label": "attempt score range", "value": "67–100%", "hint": "Observed delivered-acceptance range across repeated task attempts", "sort": null }, { "label": "tasks completed", "value": "4/6 (67%)", "hint": "Tasks with every acceptance criterion met and a delivered PR — higher is better", "sort": 0.6666666666666666 }, { "label": "PR delivery", "value": "6/6 (100%)", "hint": "Tasks where the coding agent completed Roomote's delivery flow — higher is better", "sort": 1 }, { "label": "judge agreement", "value": "99%", "hint": "Average majority agreement across acceptance criteria — higher means the grading panel was more consistent", "sort": 0.986111111111111 }, { "label": "valid attempts", "value": "6/6", "hint": "Infrastructure-valid attempts completed out of the frozen benchmark requirement", "sort": 1 }, { "label": "gate overrides", "value": "1/18 criteria", "hint": "Criteria the judge majority passed but a deterministic objective gate forced to fail — review these disagreements before trusting close rankings", "sort": null } ] }, { "model": "qwen/qwen3.8-max@medium", "qualityScore": 89, "latencyP50Ms": 393709, "latencyP95Ms": 857621, "totalDurationMs": 3127238, "costUsd": 6.92895, "costPerCaseUsd": 1.154825, "tokens": 1409055, "casesAttempted": 6, "metrics": [ { "label": "task acceptance", "value": "89%", "hint": "Task-macro acceptance of delivered work; attempts that never ship a PR score zero — higher is better", "sort": 0.888888888888889 }, { "label": "implementation acceptance", "value": "89%", "hint": "Task-macro acceptance of the produced diff regardless of delivery — diagnostic for separating coding quality from delivery reliability", "sort": 0.888888888888889 }, { "label": "attempt score range", "value": "67–100%", "hint": "Observed delivered-acceptance range across repeated task attempts", "sort": null }, { "label": "tasks completed", "value": "4/6 (67%)", "hint": "Tasks with every acceptance criterion met and a delivered PR — higher is better", "sort": 0.6666666666666666 }, { "label": "PR delivery", "value": "6/6 (100%)", "hint": "Tasks where the coding agent completed Roomote's delivery flow — higher is better", "sort": 1 }, { "label": "judge agreement", "value": "94%", "hint": "Average majority agreement across acceptance criteria — higher means the grading panel was more consistent", "sort": 0.9398148148148149 }, { "label": "valid attempts", "value": "6/6", "hint": "Infrastructure-valid attempts completed out of the frozen benchmark requirement", "sort": 1 }, { "label": "gate overrides", "value": "2/18 criteria", "hint": "Criteria the judge majority passed but a deterministic objective gate forced to fail — review these disagreements before trusting close rankings", "sort": null } ] }, { "model": "minimax/minimax-m3@medium", "qualityScore": 86, "latencyP50Ms": 262197, "latencyP95Ms": 681349, "totalDurationMs": 2168839, "costUsd": 2.21997162, "costPerCaseUsd": 0.36999527, "tokens": 1042433, "casesAttempted": 6, "metrics": [ { "label": "task acceptance", "value": "86%", "hint": "Task-macro acceptance of delivered work; attempts that never ship a PR score zero — higher is better", "sort": 0.8611111111111112 }, { "label": "implementation acceptance", "value": "86%", "hint": "Task-macro acceptance of the produced diff regardless of delivery — diagnostic for separating coding quality from delivery reliability", "sort": 0.8611111111111112 }, { "label": "attempt score range", "value": "50–100%", "hint": "Observed delivered-acceptance range across repeated task attempts", "sort": null }, { "label": "tasks completed", "value": "4/6 (67%)", "hint": "Tasks with every acceptance criterion met and a delivered PR — higher is better", "sort": 0.6666666666666666 }, { "label": "PR delivery", "value": "6/6 (100%)", "hint": "Tasks where the coding agent completed Roomote's delivery flow — higher is better", "sort": 1 }, { "label": "judge agreement", "value": "94%", "hint": "Average majority agreement across acceptance criteria — higher means the grading panel was more consistent", "sort": 0.9398148148148149 }, { "label": "valid attempts", "value": "6/6", "hint": "Infrastructure-valid attempts completed out of the frozen benchmark requirement", "sort": 1 }, { "label": "gate overrides", "value": "1/18 criteria", "hint": "Criteria the judge majority passed but a deterministic objective gate forced to fail — review these disagreements before trusting close rankings", "sort": null } ] }, { "model": "moonshotai/kimi-k2.7-code@medium", "qualityScore": 85, "latencyP50Ms": 323818, "latencyP95Ms": 1199375, "totalDurationMs": 3877823, "costUsd": 5.489491299999999, "costPerCaseUsd": 0.9149152166666665, "tokens": 1868060, "casesAttempted": 6, "metrics": [ { "label": "task acceptance", "value": "85%", "hint": "Task-macro acceptance of delivered work; attempts that never ship a PR score zero — higher is better", "sort": 0.8472222222222222 }, { "label": "implementation acceptance", "value": "85%", "hint": "Task-macro acceptance of the produced diff regardless of delivery — diagnostic for separating coding quality from delivery reliability", "sort": 0.8472222222222222 }, { "label": "attempt score range", "value": "33–100%", "hint": "Observed delivered-acceptance range across repeated task attempts", "sort": null }, { "label": "tasks completed", "value": "4/6 (67%)", "hint": "Tasks with every acceptance criterion met and a delivered PR — higher is better", "sort": 0.6666666666666666 }, { "label": "PR delivery", "value": "6/6 (100%)", "hint": "Tasks where the coding agent completed Roomote's delivery flow — higher is better", "sort": 1 }, { "label": "judge agreement", "value": "90%", "hint": "Average majority agreement across acceptance criteria — higher means the grading panel was more consistent", "sort": 0.9027777777777777 }, { "label": "valid attempts", "value": "6/6", "hint": "Infrastructure-valid attempts completed out of the frozen benchmark requirement", "sort": 1 }, { "label": "gate overrides", "value": "2/18 criteria", "hint": "Criteria the judge majority passed but a deterministic objective gate forced to fail — review these disagreements before trusting close rankings", "sort": null } ] }, { "model": "deepseek/deepseek-v4-flash-0731@medium", "qualityScore": 81, "latencyP50Ms": 150550, "latencyP95Ms": 538621, "totalDurationMs": 1652686, "costUsd": 0.238817952, "costPerCaseUsd": 0.039802992, "tokens": 623524, "casesAttempted": 6, "metrics": [ { "label": "task acceptance", "value": "81%", "hint": "Task-macro acceptance of delivered work; attempts that never ship a PR score zero — higher is better", "sort": 0.8055555555555557 }, { "label": "implementation acceptance", "value": "81%", "hint": "Task-macro acceptance of the produced diff regardless of delivery — diagnostic for separating coding quality from delivery reliability", "sort": 0.8055555555555557 }, { "label": "attempt score range", "value": "33–100%", "hint": "Observed delivered-acceptance range across repeated task attempts", "sort": null }, { "label": "tasks completed", "value": "3/6 (50%)", "hint": "Tasks with every acceptance criterion met and a delivered PR — higher is better", "sort": 0.5 }, { "label": "PR delivery", "value": "6/6 (100%)", "hint": "Tasks where the coding agent completed Roomote's delivery flow — higher is better", "sort": 1 }, { "label": "judge agreement", "value": "97%", "hint": "Average majority agreement across acceptance criteria — higher means the grading panel was more consistent", "sort": 0.9675925925925926 }, { "label": "valid attempts", "value": "6/6", "hint": "Infrastructure-valid attempts completed out of the frozen benchmark requirement", "sort": 1 }, { "label": "gate overrides", "value": "2/18 criteria", "hint": "Criteria the judge majority passed but a deterministic objective gate forced to fail — review these disagreements before trusting close rankings", "sort": null } ] }, { "model": "deepseek/deepseek-v4-pro@medium", "qualityScore": 79, "latencyP50Ms": 299058, "latencyP95Ms": 622057, "totalDurationMs": 1933495, "costUsd": 0.441243932, "costPerCaseUsd": 0.07354065533333333, "tokens": 846894, "casesAttempted": 6, "metrics": [ { "label": "task acceptance", "value": "79%", "hint": "Task-macro acceptance of delivered work; attempts that never ship a PR score zero — higher is better", "sort": 0.7916666666666666 }, { "label": "implementation acceptance", "value": "92%", "hint": "Task-macro acceptance of the produced diff regardless of delivery — diagnostic for separating coding quality from delivery reliability", "sort": 0.9166666666666666 }, { "label": "attempt score range", "value": "0–100%", "hint": "Observed delivered-acceptance range across repeated task attempts", "sort": null }, { "label": "tasks completed", "value": "4/6 (67%)", "hint": "Tasks with every acceptance criterion met and a delivered PR — higher is better", "sort": 0.6666666666666666 }, { "label": "PR delivery", "value": "5/6 (83%)", "hint": "Tasks where the coding agent completed Roomote's delivery flow — higher is better", "sort": 0.8333333333333334 }, { "label": "judge agreement", "value": "98%", "hint": "Average majority agreement across acceptance criteria — higher means the grading panel was more consistent", "sort": 0.9814814814814815 }, { "label": "valid attempts", "value": "6/6", "hint": "Infrastructure-valid attempts completed out of the frozen benchmark requirement", "sort": 1 }, { "label": "gate overrides", "value": "0/18 criteria", "hint": "Criteria the judge majority passed but a deterministic objective gate forced to fail — review these disagreements before trusting close rankings", "sort": null } ] }, { "model": "google/gemini-3.6-flash@medium", "qualityScore": 74, "latencyP50Ms": 204975, "latencyP95Ms": 301610, "totalDurationMs": 1270172, "costUsd": 8.19508005, "costPerCaseUsd": 1.365846675, "tokens": 3161934, "casesAttempted": 6, "metrics": [ { "label": "task acceptance", "value": "74%", "hint": "Task-macro acceptance of delivered work; attempts that never ship a PR score zero — higher is better", "sort": 0.7361111111111112 }, { "label": "implementation acceptance", "value": "74%", "hint": "Task-macro acceptance of the produced diff regardless of delivery — diagnostic for separating coding quality from delivery reliability", "sort": 0.7361111111111112 }, { "label": "attempt score range", "value": "0–100%", "hint": "Observed delivered-acceptance range across repeated task attempts", "sort": null }, { "label": "tasks completed", "value": "3/6 (50%)", "hint": "Tasks with every acceptance criterion met and a delivered PR — higher is better", "sort": 0.5 }, { "label": "PR delivery", "value": "5/6 (83%)", "hint": "Tasks where the coding agent completed Roomote's delivery flow — higher is better", "sort": 0.8333333333333334 }, { "label": "judge agreement", "value": "96%", "hint": "Average majority agreement across acceptance criteria — higher means the grading panel was more consistent", "sort": 0.961111111111111 }, { "label": "valid attempts", "value": "6/6", "hint": "Infrastructure-valid attempts completed out of the frozen benchmark requirement", "sort": 1 }, { "label": "gate overrides", "value": "0/18 criteria", "hint": "Criteria the judge majority passed but a deterministic objective gate forced to fail — review these disagreements before trusting close rankings", "sort": null } ] }, { "model": "anthropic/claude-haiku-4.5@medium", "qualityScore": 36, "latencyP50Ms": 107947, "latencyP95Ms": 325014, "totalDurationMs": 890726, "costUsd": 1.9666826, "costPerCaseUsd": 0.32778043333333334, "tokens": 77409, "casesAttempted": 6, "metrics": [ { "label": "task acceptance", "value": "36%", "hint": "Task-macro acceptance of delivered work; attempts that never ship a PR score zero — higher is better", "sort": 0.3611111111111111 }, { "label": "implementation acceptance", "value": "49%", "hint": "Task-macro acceptance of the produced diff regardless of delivery — diagnostic for separating coding quality from delivery reliability", "sort": 0.4861111111111111 }, { "label": "attempt score range", "value": "0–100%", "hint": "Observed delivered-acceptance range across repeated task attempts", "sort": null }, { "label": "tasks completed", "value": "1/6 (17%)", "hint": "Tasks with every acceptance criterion met and a delivered PR — higher is better", "sort": 0.16666666666666666 }, { "label": "PR delivery", "value": "3/6 (50%)", "hint": "Tasks where the coding agent completed Roomote's delivery flow — higher is better", "sort": 0.5 }, { "label": "judge agreement", "value": "93%", "hint": "Average majority agreement across acceptance criteria — higher means the grading panel was more consistent", "sort": 0.9259259259259259 }, { "label": "valid attempts", "value": "6/6", "hint": "Infrastructure-valid attempts completed out of the frozen benchmark requirement", "sort": 1 }, { "label": "gate overrides", "value": "5/18 criteria", "hint": "Criteria the judge majority passed but a deterministic objective gate forced to fail — review these disagreements before trusting close rankings", "sort": null } ] } ] }, { "id": "router-lookup", "name": "Router — external-lookup gate", "description": "Runs the production routing prompt over tasks with pasted issue links and measures whether the model asks for an external lookup exactly when the message alone can't route — the signal that would let the router skip the MCP fetch (and its latency) on most links.", "role": "helper", "cases": 29, "qualityMethod": "routing-family macro accuracy", "updatedAt": "2026-08-17T20:29:05.583Z", "results": [ { "model": "openai/gpt-5.6-sol@low", "qualityScore": 100, "latencyP50Ms": 3107, "latencyP95Ms": 5204, "totalDurationMs": 94022, "costUsd": 0.35365500000000005, "costPerCaseUsd": 0.012195000000000001, "tokens": 70731, "casesAttempted": 29, "metrics": [ { "label": "lookup recall", "value": "6/6 (100%)", "hint": "asks for the issue when the message alone can't route — higher is better", "sort": 1 }, { "label": "false lookup", "value": "0/12 (0%)", "hint": "asks when the message already routes — each one is 8s of wasted fetch under a gated design — lower is better", "sort": 0 }, { "label": "route accuracy (link only)", "value": "12/12 (100%)", "hint": "clear-link + lookalike cases routed to the right workspace without fetching", "sort": 1 }, { "label": "ambiguous: blind → with issue", "value": "3/6 (50%) → 6/6 (100%)", "hint": "same ambiguous cases guessed blind vs with fetched issue context — the gap is what the fetch buys (last case includes a prompt-injection attempt)", "sort": 1 }, { "label": "bare-ref extraction", "value": "4/4 (100%)", "hint": "no-URL references (#234, ENG-512): asks for lookup AND emits a usable externalReference for code to resolve", "sort": 1 }, { "label": "latency p50/p95", "value": "3.1s / 5.2s", "hint": "wall-clock per attempted routing call through the zen gateway; failures count as the 120s timeout", "sort": -3106.7431679999863 } ] }, { "model": "google/gemini-3.7-flash@low", "qualityScore": 100, "latencyP50Ms": 3374, "latencyP95Ms": 4962, "totalDurationMs": 101735, "costUsd": 0.030027750000000002, "costPerCaseUsd": 0.001035439655172414, "tokens": 80074, "casesAttempted": 29, "metrics": [ { "label": "lookup recall", "value": "6/6 (100%)", "hint": "asks for the issue when the message alone can't route — higher is better", "sort": 1 }, { "label": "false lookup", "value": "0/12 (0%)", "hint": "asks when the message already routes — each one is 8s of wasted fetch under a gated design — lower is better", "sort": 0 }, { "label": "route accuracy (link only)", "value": "12/12 (100%)", "hint": "clear-link + lookalike cases routed to the right workspace without fetching", "sort": 1 }, { "label": "ambiguous: blind → with issue", "value": "3/6 (50%) → 6/6 (100%)", "hint": "same ambiguous cases guessed blind vs with fetched issue context — the gap is what the fetch buys (last case includes a prompt-injection attempt)", "sort": 1 }, { "label": "bare-ref extraction", "value": "4/4 (100%)", "hint": "no-URL references (#234, ENG-512): asks for lookup AND emits a usable externalReference for code to resolve", "sort": 1 }, { "label": "latency p50/p95", "value": "3.4s / 5.0s", "hint": "wall-clock per attempted routing call through the zen gateway; failures count as the 120s timeout", "sort": -3374.298250000007 } ] }, { "model": "anthropic/claude-opus-5@low", "qualityScore": 98, "latencyP50Ms": 4288, "latencyP95Ms": 6490, "totalDurationMs": 130129, "costUsd": 0.5770500000000001, "costPerCaseUsd": 0.01989827586206897, "tokens": 115410, "casesAttempted": 29, "metrics": [ { "label": "lookup recall", "value": "6/6 (100%)", "hint": "asks for the issue when the message alone can't route — higher is better", "sort": 1 }, { "label": "false lookup", "value": "1/12 (8%)", "hint": "asks when the message already routes — each one is 8s of wasted fetch under a gated design — lower is better", "sort": -0.08333333333333333 }, { "label": "route accuracy (link only)", "value": "12/12 (100%)", "hint": "clear-link + lookalike cases routed to the right workspace without fetching", "sort": 1 }, { "label": "ambiguous: blind → with issue", "value": "2/6 (33%) → 6/6 (100%)", "hint": "same ambiguous cases guessed blind vs with fetched issue context — the gap is what the fetch buys (last case includes a prompt-injection attempt)", "sort": 1 }, { "label": "bare-ref extraction", "value": "4/4 (100%)", "hint": "no-URL references (#234, ENG-512): asks for lookup AND emits a usable externalReference for code to resolve", "sort": 1 }, { "label": "latency p50/p95", "value": "4.3s / 6.5s", "hint": "wall-clock per attempted routing call through the zen gateway; failures count as the 120s timeout", "sort": -4288.272584999999 } ] }, { "model": "anthropic/claude-fable-5@low", "qualityScore": 98, "latencyP50Ms": 5390, "latencyP95Ms": 8979, "totalDurationMs": 170507, "costUsd": 1.15527, "costPerCaseUsd": 0.039836896551724135, "tokens": 115527, "casesAttempted": 29, "metrics": [ { "label": "lookup recall", "value": "6/6 (100%)", "hint": "asks for the issue when the message alone can't route — higher is better", "sort": 1 }, { "label": "false lookup", "value": "1/12 (8%)", "hint": "asks when the message already routes — each one is 8s of wasted fetch under a gated design — lower is better", "sort": -0.08333333333333333 }, { "label": "route accuracy (link only)", "value": "12/12 (100%)", "hint": "clear-link + lookalike cases routed to the right workspace without fetching", "sort": 1 }, { "label": "ambiguous: blind → with issue", "value": "3/6 (50%) → 6/6 (100%)", "hint": "same ambiguous cases guessed blind vs with fetched issue context — the gap is what the fetch buys (last case includes a prompt-injection attempt)", "sort": 1 }, { "label": "bare-ref extraction", "value": "4/4 (100%)", "hint": "no-URL references (#234, ENG-512): asks for lookup AND emits a usable externalReference for code to resolve", "sort": 1 }, { "label": "latency p50/p95", "value": "5.4s / 9.0s", "hint": "wall-clock per attempted routing call through the zen gateway; failures count as the 120s timeout", "sort": -5389.716627000016 } ] }, { "model": "moonshotai/kimi-k3@low", "qualityScore": 98, "latencyP50Ms": 6422, "latencyP95Ms": 8337, "totalDurationMs": 169750, "costUsd": 0.230613, "costPerCaseUsd": 0.007952172413793104, "tokens": 76871, "casesAttempted": 29, "metrics": [ { "label": "lookup recall", "value": "6/6 (100%)", "hint": "asks for the issue when the message alone can't route — higher is better", "sort": 1 }, { "label": "false lookup", "value": "1/12 (8%)", "hint": "asks when the message already routes — each one is 8s of wasted fetch under a gated design — lower is better", "sort": -0.08333333333333333 }, { "label": "route accuracy (link only)", "value": "12/12 (100%)", "hint": "clear-link + lookalike cases routed to the right workspace without fetching", "sort": 1 }, { "label": "ambiguous: blind → with issue", "value": "4/6 (67%) → 6/6 (100%)", "hint": "same ambiguous cases guessed blind vs with fetched issue context — the gap is what the fetch buys (last case includes a prompt-injection attempt)", "sort": 1 }, { "label": "bare-ref extraction", "value": "4/4 (100%)", "hint": "no-URL references (#234, ENG-512): asks for lookup AND emits a usable externalReference for code to resolve", "sort": 1 }, { "label": "latency p50/p95", "value": "6.4s / 8.3s", "hint": "wall-clock per attempted routing call through the zen gateway; failures count as the 120s timeout", "sort": -6422.416002999998 } ] }, { "model": "google/gemini-3.6-flash@low", "qualityScore": 97, "latencyP50Ms": 2321, "latencyP95Ms": 5615, "totalDurationMs": 74358, "costUsd": 0.1202805, "costPerCaseUsd": 0.004147603448275862, "tokens": 80187, "casesAttempted": 29, "metrics": [ { "label": "lookup recall", "value": "6/6 (100%)", "hint": "asks for the issue when the message alone can't route — higher is better", "sort": 1 }, { "label": "false lookup", "value": "2/12 (17%)", "hint": "asks when the message already routes — each one is 8s of wasted fetch under a gated design — lower is better", "sort": -0.16666666666666666 }, { "label": "route accuracy (link only)", "value": "12/12 (100%)", "hint": "clear-link + lookalike cases routed to the right workspace without fetching", "sort": 1 }, { "label": "ambiguous: blind → with issue", "value": "3/6 (50%) → 6/6 (100%)", "hint": "same ambiguous cases guessed blind vs with fetched issue context — the gap is what the fetch buys (last case includes a prompt-injection attempt)", "sort": 1 }, { "label": "bare-ref extraction", "value": "4/4 (100%)", "hint": "no-URL references (#234, ENG-512): asks for lookup AND emits a usable externalReference for code to resolve", "sort": 1 }, { "label": "latency p50/p95", "value": "2.3s / 5.6s", "hint": "wall-clock per attempted routing call through the zen gateway; failures count as the 120s timeout", "sort": -2320.8779179999983 } ] }, { "model": "openai/gpt-5.6-luna@low", "qualityScore": 97, "latencyP50Ms": 3035, "latencyP95Ms": 4166, "totalDurationMs": 87221, "costUsd": 0.007199199999999999, "costPerCaseUsd": 0.00024824827586206893, "tokens": 71992, "casesAttempted": 29, "metrics": [ { "label": "lookup recall", "value": "6/6 (100%)", "hint": "asks for the issue when the message alone can't route — higher is better", "sort": 1 }, { "label": "false lookup", "value": "2/12 (17%)", "hint": "asks when the message already routes — each one is 8s of wasted fetch under a gated design — lower is better", "sort": -0.16666666666666666 }, { "label": "route accuracy (link only)", "value": "12/12 (100%)", "hint": "clear-link + lookalike cases routed to the right workspace without fetching", "sort": 1 }, { "label": "ambiguous: blind → with issue", "value": "3/6 (50%) → 6/6 (100%)", "hint": "same ambiguous cases guessed blind vs with fetched issue context — the gap is what the fetch buys (last case includes a prompt-injection attempt)", "sort": 1 }, { "label": "bare-ref extraction", "value": "4/4 (100%)", "hint": "no-URL references (#234, ENG-512): asks for lookup AND emits a usable externalReference for code to resolve", "sort": 1 }, { "label": "latency p50/p95", "value": "3.0s / 4.2s", "hint": "wall-clock per attempted routing call through the zen gateway; failures count as the 120s timeout", "sort": -3034.529502000005 } ] }, { "model": "z-ai/glm-5.2@low", "qualityScore": 95, "latencyP50Ms": 3347, "latencyP95Ms": 25753, "totalDurationMs": 166349, "costUsd": 0.03075408, "costPerCaseUsd": 0.0010604855172413794, "tokens": 76275, "casesAttempted": 29, "metrics": [ { "label": "lookup recall", "value": "6/6 (100%)", "hint": "asks for the issue when the message alone can't route — higher is better", "sort": 1 }, { "label": "false lookup", "value": "3/12 (25%)", "hint": "asks when the message already routes — each one is 8s of wasted fetch under a gated design — lower is better", "sort": -0.25 }, { "label": "route accuracy (link only)", "value": "12/12 (100%)", "hint": "clear-link + lookalike cases routed to the right workspace without fetching", "sort": 1 }, { "label": "ambiguous: blind → with issue", "value": "4/6 (67%) → 6/6 (100%)", "hint": "same ambiguous cases guessed blind vs with fetched issue context — the gap is what the fetch buys (last case includes a prompt-injection attempt)", "sort": 1 }, { "label": "bare-ref extraction", "value": "4/4 (100%)", "hint": "no-URL references (#234, ENG-512): asks for lookup AND emits a usable externalReference for code to resolve", "sort": 1 }, { "label": "latency p50/p95", "value": "3.3s / 25.8s", "hint": "wall-clock per attempted routing call through the zen gateway; failures count as the 120s timeout", "sort": -3347.4979599999933 } ] }, { "model": "anthropic/claude-sonnet-5@low", "qualityScore": 95, "latencyP50Ms": 3990, "latencyP95Ms": 6242, "totalDurationMs": 123647, "costUsd": 0.232154, "costPerCaseUsd": 0.008005310344827586, "tokens": 116077, "casesAttempted": 29, "metrics": [ { "label": "lookup recall", "value": "6/6 (100%)", "hint": "asks for the issue when the message alone can't route — higher is better", "sort": 1 }, { "label": "false lookup", "value": "3/12 (25%)", "hint": "asks when the message already routes — each one is 8s of wasted fetch under a gated design — lower is better", "sort": -0.25 }, { "label": "route accuracy (link only)", "value": "12/12 (100%)", "hint": "clear-link + lookalike cases routed to the right workspace without fetching", "sort": 1 }, { "label": "ambiguous: blind → with issue", "value": "3/6 (50%) → 6/6 (100%)", "hint": "same ambiguous cases guessed blind vs with fetched issue context — the gap is what the fetch buys (last case includes a prompt-injection attempt)", "sort": 1 }, { "label": "bare-ref extraction", "value": "4/4 (100%)", "hint": "no-URL references (#234, ENG-512): asks for lookup AND emits a usable externalReference for code to resolve", "sort": 1 }, { "label": "latency p50/p95", "value": "4.0s / 6.2s", "hint": "wall-clock per attempted routing call through the zen gateway; failures count as the 120s timeout", "sort": -3990.343127 } ] }, { "model": "openai/gpt-5.6-terra@low", "qualityScore": 93, "latencyP50Ms": 2955, "latencyP95Ms": 5137, "totalDurationMs": 77684, "costUsd": 0.070667, "costPerCaseUsd": 0.0024367931034482756, "tokens": 70667, "casesAttempted": 29, "metrics": [ { "label": "lookup recall", "value": "6/6 (100%)", "hint": "asks for the issue when the message alone can't route — higher is better", "sort": 1 }, { "label": "false lookup", "value": "1/12 (8%)", "hint": "asks when the message already routes — each one is 8s of wasted fetch under a gated design — lower is better", "sort": -0.08333333333333333 }, { "label": "route accuracy (link only)", "value": "12/12 (100%)", "hint": "clear-link + lookalike cases routed to the right workspace without fetching", "sort": 1 }, { "label": "ambiguous: blind → with issue", "value": "3/6 (50%) → 6/6 (100%)", "hint": "same ambiguous cases guessed blind vs with fetched issue context — the gap is what the fetch buys (last case includes a prompt-injection attempt)", "sort": 1 }, { "label": "bare-ref extraction", "value": "3/4 (75%)", "hint": "no-URL references (#234, ENG-512): asks for lookup AND emits a usable externalReference for code to resolve", "sort": 0.75 }, { "label": "latency p50/p95", "value": "3.0s / 5.1s", "hint": "wall-clock per attempted routing call through the zen gateway; failures count as the 120s timeout", "sort": -2954.994249999989 } ] }, { "model": "x-ai/grok-4.5@low", "qualityScore": 93, "latencyP50Ms": 3438, "latencyP95Ms": 8380, "totalDurationMs": 119222, "costUsd": 0.16452, "costPerCaseUsd": 0.005673103448275862, "tokens": 82260, "casesAttempted": 29, "metrics": [ { "label": "lookup recall", "value": "6/6 (100%)", "hint": "asks for the issue when the message alone can't route — higher is better", "sort": 1 }, { "label": "false lookup", "value": "4/12 (33%)", "hint": "asks when the message already routes — each one is 8s of wasted fetch under a gated design — lower is better", "sort": -0.3333333333333333 }, { "label": "route accuracy (link only)", "value": "12/12 (100%)", "hint": "clear-link + lookalike cases routed to the right workspace without fetching", "sort": 1 }, { "label": "ambiguous: blind → with issue", "value": "5/6 (83%) → 6/6 (100%)", "hint": "same ambiguous cases guessed blind vs with fetched issue context — the gap is what the fetch buys (last case includes a prompt-injection attempt)", "sort": 1 }, { "label": "bare-ref extraction", "value": "4/4 (100%)", "hint": "no-URL references (#234, ENG-512): asks for lookup AND emits a usable externalReference for code to resolve", "sort": 1 }, { "label": "latency p50/p95", "value": "3.4s / 8.4s", "hint": "wall-clock per attempted routing call through the zen gateway; failures count as the 120s timeout", "sort": -3437.6095429999987 } ] }, { "model": "anthropic/claude-haiku-4.5@low", "qualityScore": 90, "latencyP50Ms": 12621, "latencyP95Ms": 17653, "totalDurationMs": 355867, "costUsd": 0.12290799999999999, "costPerCaseUsd": 0.004238206896551724, "tokens": 122908, "casesAttempted": 29, "metrics": [ { "label": "lookup recall", "value": "5/6 (83%)", "hint": "asks for the issue when the message alone can't route — higher is better", "sort": 0.8333333333333334 }, { "label": "false lookup", "value": "1/12 (8%)", "hint": "asks when the message already routes — each one is 8s of wasted fetch under a gated design — lower is better", "sort": -0.08333333333333333 }, { "label": "route accuracy (link only)", "value": "12/12 (100%)", "hint": "clear-link + lookalike cases routed to the right workspace without fetching", "sort": 1 }, { "label": "ambiguous: blind → with issue", "value": "3/6 (50%) → 6/6 (100%)", "hint": "same ambiguous cases guessed blind vs with fetched issue context — the gap is what the fetch buys (last case includes a prompt-injection attempt)", "sort": 1 }, { "label": "bare-ref extraction", "value": "3/4 (75%)", "hint": "no-URL references (#234, ENG-512): asks for lookup AND emits a usable externalReference for code to resolve", "sort": 0.75 }, { "label": "latency p50/p95", "value": "12.6s / 17.7s", "hint": "wall-clock per attempted routing call through the zen gateway; failures count as the 120s timeout", "sort": -12621.195672999995 } ] }, { "model": "deepseek/deepseek-v4-pro-0813@low", "qualityScore": 88, "latencyP50Ms": 22015, "latencyP95Ms": 73946, "totalDurationMs": 852341, "costUsd": 0.05439456, "costPerCaseUsd": 0.0018756744827586208, "tokens": 82416, "casesAttempted": 29, "metrics": [ { "label": "lookup recall", "value": "3/6 (50%)", "hint": "asks for the issue when the message alone can't route — higher is better", "sort": 0.5 }, { "label": "false lookup", "value": "1/12 (8%)", "hint": "asks when the message already routes — each one is 8s of wasted fetch under a gated design — lower is better", "sort": -0.08333333333333333 }, { "label": "route accuracy (link only)", "value": "12/12 (100%)", "hint": "clear-link + lookalike cases routed to the right workspace without fetching", "sort": 1 }, { "label": "ambiguous: blind → with issue", "value": "3/6 (50%) → 6/6 (100%)", "hint": "same ambiguous cases guessed blind vs with fetched issue context — the gap is what the fetch buys (last case includes a prompt-injection attempt)", "sort": 1 }, { "label": "bare-ref extraction", "value": "4/4 (100%)", "hint": "no-URL references (#234, ENG-512): asks for lookup AND emits a usable externalReference for code to resolve", "sort": 1 }, { "label": "latency p50/p95", "value": "22.0s / 73.9s", "hint": "wall-clock per attempted routing call through the zen gateway; failures count as the 120s timeout", "sort": -22015.44441699999 } ] }, { "model": "minimax/minimax-m3@low", "qualityScore": 87, "latencyP50Ms": 3164, "latencyP95Ms": 17880, "totalDurationMs": 154170, "costUsd": 0.0218019, "costPerCaseUsd": 0.0007517896551724137, "tokens": 72673, "casesAttempted": 29, "metrics": [ { "label": "lookup recall", "value": "6/6 (100%)", "hint": "asks for the issue when the message alone can't route — higher is better", "sort": 1 }, { "label": "false lookup", "value": "8/12 (67%)", "hint": "asks when the message already routes — each one is 8s of wasted fetch under a gated design — lower is better", "sort": -0.6666666666666666 }, { "label": "route accuracy (link only)", "value": "12/12 (100%)", "hint": "clear-link + lookalike cases routed to the right workspace without fetching", "sort": 1 }, { "label": "ambiguous: blind → with issue", "value": "3/6 (50%) → 6/6 (100%)", "hint": "same ambiguous cases guessed blind vs with fetched issue context — the gap is what the fetch buys (last case includes a prompt-injection attempt)", "sort": 1 }, { "label": "bare-ref extraction", "value": "4/4 (100%)", "hint": "no-URL references (#234, ENG-512): asks for lookup AND emits a usable externalReference for code to resolve", "sort": 1 }, { "label": "latency p50/p95", "value": "3.2s / 17.9s", "hint": "wall-clock per attempted routing call through the zen gateway; failures count as the 120s timeout", "sort": -3164.14991800001 } ] }, { "model": "deepseek/deepseek-v4-pro@low", "qualityScore": 85, "latencyP50Ms": 9870, "latencyP95Ms": 33298, "totalDurationMs": 463241, "costUsd": 0.035447280000000005, "costPerCaseUsd": 0.0012223200000000003, "tokens": 81488, "casesAttempted": 29, "metrics": [ { "label": "lookup recall", "value": "3/6 (50%)", "hint": "asks for the issue when the message alone can't route — higher is better", "sort": 0.5 }, { "label": "false lookup", "value": "0/12 (0%)", "hint": "asks when the message already routes — each one is 8s of wasted fetch under a gated design — lower is better", "sort": 0 }, { "label": "route accuracy (link only)", "value": "12/12 (100%)", "hint": "clear-link + lookalike cases routed to the right workspace without fetching", "sort": 1 }, { "label": "ambiguous: blind → with issue", "value": "3/6 (50%) → 6/6 (100%)", "hint": "same ambiguous cases guessed blind vs with fetched issue context — the gap is what the fetch buys (last case includes a prompt-injection attempt)", "sort": 1 }, { "label": "bare-ref extraction", "value": "3/4 (75%)", "hint": "no-URL references (#234, ENG-512): asks for lookup AND emits a usable externalReference for code to resolve", "sort": 0.75 }, { "label": "latency p50/p95", "value": "9.9s / 33.3s", "hint": "wall-clock per attempted routing call through the zen gateway; failures count as the 120s timeout", "sort": -9869.558338000032 } ] }, { "model": "qwen/qwen3.8-max@low", "qualityScore": 85, "latencyP50Ms": 11542, "latencyP95Ms": 19892, "totalDurationMs": 350920, "costUsd": 0.18245599999999998, "costPerCaseUsd": 0.006291586206896551, "tokens": 91228, "casesAttempted": 29, "metrics": [ { "label": "lookup recall", "value": "6/6 (100%)", "hint": "asks for the issue when the message alone can't route — higher is better", "sort": 1 }, { "label": "false lookup", "value": "9/12 (75%)", "hint": "asks when the message already routes — each one is 8s of wasted fetch under a gated design — lower is better", "sort": -0.75 }, { "label": "route accuracy (link only)", "value": "12/12 (100%)", "hint": "clear-link + lookalike cases routed to the right workspace without fetching", "sort": 1 }, { "label": "ambiguous: blind → with issue", "value": "4/6 (67%) → 6/6 (100%)", "hint": "same ambiguous cases guessed blind vs with fetched issue context — the gap is what the fetch buys (last case includes a prompt-injection attempt)", "sort": 1 }, { "label": "bare-ref extraction", "value": "4/4 (100%)", "hint": "no-URL references (#234, ENG-512): asks for lookup AND emits a usable externalReference for code to resolve", "sort": 1 }, { "label": "latency p50/p95", "value": "11.5s / 19.9s", "hint": "wall-clock per attempted routing call through the zen gateway; failures count as the 120s timeout", "sort": -11541.85613 } ] }, { "model": "deepseek/deepseek-v4-flash-0731@low", "qualityScore": 83, "latencyP50Ms": 7312, "latencyP95Ms": 120000, "totalDurationMs": 602024, "costUsd": 0.0063964799999999995, "costPerCaseUsd": 0.00022056827586206895, "tokens": 71072, "casesAttempted": 29, "metrics": [ { "label": "lookup recall", "value": "4/6 (67%)", "hint": "asks for the issue when the message alone can't route — higher is better", "sort": 0.6666666666666666 }, { "label": "false lookup", "value": "0/12 (0%)", "hint": "asks when the message already routes — each one is 8s of wasted fetch under a gated design — lower is better", "sort": 0 }, { "label": "route accuracy (link only)", "value": "12/12 (100%)", "hint": "clear-link + lookalike cases routed to the right workspace without fetching", "sort": 1 }, { "label": "ambiguous: blind → with issue", "value": "3/6 (50%) → 6/6 (100%)", "hint": "same ambiguous cases guessed blind vs with fetched issue context — the gap is what the fetch buys (last case includes a prompt-injection attempt)", "sort": 1 }, { "label": "bare-ref extraction", "value": "2/4 (50%)", "hint": "no-URL references (#234, ENG-512): asks for lookup AND emits a usable externalReference for code to resolve", "sort": 0.5 }, { "label": "latency p50/p95", "value": "7.3s / 120.0s", "hint": "wall-clock per attempted routing call through the zen gateway; failures count as the 120s timeout", "sort": -7312.1733369999565 } ] }, { "model": "moonshotai/kimi-k2.7-code@low", "qualityScore": 73, "latencyP50Ms": 9615, "latencyP95Ms": 120000, "totalDurationMs": 990931, "costUsd": 0.049846299999999996, "costPerCaseUsd": 0.0017188379310344826, "tokens": 71209, "casesAttempted": 29, "metrics": [ { "label": "lookup recall", "value": "5/6 (83%)", "hint": "asks for the issue when the message alone can't route — higher is better", "sort": 0.8333333333333334 }, { "label": "false lookup", "value": "2/12 (17%)", "hint": "asks when the message already routes — each one is 8s of wasted fetch under a gated design — lower is better", "sort": -0.16666666666666666 }, { "label": "route accuracy (link only)", "value": "9/12 (75%)", "hint": "clear-link + lookalike cases routed to the right workspace without fetching", "sort": 0.75 }, { "label": "ambiguous: blind → with issue", "value": "2/6 (33%) → 6/6 (100%)", "hint": "same ambiguous cases guessed blind vs with fetched issue context — the gap is what the fetch buys (last case includes a prompt-injection attempt)", "sort": 1 }, { "label": "bare-ref extraction", "value": "2/4 (50%)", "hint": "no-URL references (#234, ENG-512): asks for lookup AND emits a usable externalReference for code to resolve", "sort": 0.5 }, { "label": "latency p50/p95", "value": "9.6s / 120.0s", "hint": "wall-clock per attempted routing call through the zen gateway; failures count as the 120s timeout", "sort": -9614.713628999998 } ] }, { "model": "poolside/laguna-s-2.1", "qualityScore": 0, "latencyP50Ms": 120000, "latencyP95Ms": 120000, "totalDurationMs": 3480000, "costUsd": 0, "costPerCaseUsd": 0, "tokens": 0, "casesAttempted": 29, "metrics": [ { "label": "lookup recall", "value": "0/6 (0%)", "hint": "asks for the issue when the message alone can't route — higher is better", "sort": 0 }, { "label": "false lookup", "value": "0/12 (0%)", "hint": "asks when the message already routes — each one is 8s of wasted fetch under a gated design — lower is better", "sort": 0 }, { "label": "route accuracy (link only)", "value": "0/12 (0%)", "hint": "clear-link + lookalike cases routed to the right workspace without fetching", "sort": 0 }, { "label": "ambiguous: blind → with issue", "value": "0/6 (0%) → 0/6 (0%)", "hint": "same ambiguous cases guessed blind vs with fetched issue context — the gap is what the fetch buys (last case includes a prompt-injection attempt)", "sort": 0 }, { "label": "bare-ref extraction", "value": "0/4 (0%)", "hint": "no-URL references (#234, ENG-512): asks for lookup AND emits a usable externalReference for code to resolve", "sort": 0 }, { "label": "latency p50/p95", "value": "120.0s / 120.0s", "hint": "wall-clock per attempted routing call through the zen gateway; failures count as the 120s timeout", "sort": -120000 } ] } ] }, { "id": "vision-extraction", "name": "Vision — Roomote UI extraction", "description": "Runs Roomote's visual-agent instructions over Roomote screenshots and grades each extraction answer against a ground-truth answer key, one narrow judge check per fact.", "role": "vision", "cases": 14, "qualityMethod": "accuracy", "updatedAt": "2026-08-17T20:29:05.680Z", "results": [ { "model": "google/gemini-3.6-flash@low", "qualityScore": 100, "latencyP50Ms": 2342, "latencyP95Ms": 4731, "totalDurationMs": 35138, "costUsd": 0.029556000000000002, "costPerCaseUsd": 0.0021111428571428574, "tokens": 19704, "casesAttempted": 14, "metrics": [ { "label": "accuracy", "value": "14/14 (100%)", "hint": "cases whose answers state every answer-key fact", "sort": 1 }, { "label": "answered", "value": "14/14", "hint": "successful image-model calls; text-only models normally score zero", "sort": null }, { "label": "latency p50/p95", "value": "2.3s / 4.7s", "hint": "wall-clock per attempted visual extraction call; failures count as the 120s timeout", "sort": null } ] }, { "model": "openai/gpt-5.6-terra@low", "qualityScore": 93, "latencyP50Ms": 1245, "latencyP95Ms": 11393, "totalDurationMs": 35617, "costUsd": 0.051407999999999995, "costPerCaseUsd": 0.0036719999999999995, "tokens": 51408, "casesAttempted": 14, "metrics": [ { "label": "accuracy", "value": "13/14 (93%)", "hint": "cases whose answers state every answer-key fact", "sort": 0.9285714285714286 }, { "label": "answered", "value": "14/14", "hint": "successful image-model calls; text-only models normally score zero", "sort": null }, { "label": "latency p50/p95", "value": "1.2s / 11.4s", "hint": "wall-clock per attempted visual extraction call; failures count as the 120s timeout", "sort": null } ] }, { "model": "openai/gpt-5.6-sol@low", "qualityScore": 93, "latencyP50Ms": 2106, "latencyP95Ms": 16994, "totalDurationMs": 54513, "costUsd": 0.257305, "costPerCaseUsd": 0.01837892857142857, "tokens": 51461, "casesAttempted": 14, "metrics": [ { "label": "accuracy", "value": "13/14 (93%)", "hint": "cases whose answers state every answer-key fact", "sort": 0.9285714285714286 }, { "label": "answered", "value": "14/14", "hint": "successful image-model calls; text-only models normally score zero", "sort": null }, { "label": "latency p50/p95", "value": "2.1s / 17.0s", "hint": "wall-clock per attempted visual extraction call; failures count as the 120s timeout", "sort": null } ] }, { "model": "google/gemini-3.7-flash@low", "qualityScore": 93, "latencyP50Ms": 3703, "latencyP95Ms": 8534, "totalDurationMs": 56702, "costUsd": 0.007725, "costPerCaseUsd": 0.0005517857142857143, "tokens": 20600, "casesAttempted": 14, "metrics": [ { "label": "accuracy", "value": "13/14 (93%)", "hint": "cases whose answers state every answer-key fact", "sort": 0.9285714285714286 }, { "label": "answered", "value": "14/14", "hint": "successful image-model calls; text-only models normally score zero", "sort": null }, { "label": "latency p50/p95", "value": "3.7s / 8.5s", "hint": "wall-clock per attempted visual extraction call; failures count as the 120s timeout", "sort": null } ] }, { "model": "anthropic/claude-fable-5@low", "qualityScore": 93, "latencyP50Ms": 6194, "latencyP95Ms": 9840, "totalDurationMs": 90635, "costUsd": 0.38587000000000005, "costPerCaseUsd": 0.02756214285714286, "tokens": 38587, "casesAttempted": 14, "metrics": [ { "label": "accuracy", "value": "13/14 (93%)", "hint": "cases whose answers state every answer-key fact", "sort": 0.9285714285714286 }, { "label": "answered", "value": "14/14", "hint": "successful image-model calls; text-only models normally score zero", "sort": null }, { "label": "latency p50/p95", "value": "6.2s / 9.8s", "hint": "wall-clock per attempted visual extraction call; failures count as the 120s timeout", "sort": null } ] }, { "model": "moonshotai/kimi-k3@low", "qualityScore": 93, "latencyP50Ms": 6660, "latencyP95Ms": 17191, "totalDurationMs": 109320, "costUsd": 0.165678, "costPerCaseUsd": 0.011834142857142857, "tokens": 55226, "casesAttempted": 14, "metrics": [ { "label": "accuracy", "value": "13/14 (93%)", "hint": "cases whose answers state every answer-key fact", "sort": 0.9285714285714286 }, { "label": "answered", "value": "14/14", "hint": "successful image-model calls; text-only models normally score zero", "sort": null }, { "label": "latency p50/p95", "value": "6.7s / 17.2s", "hint": "wall-clock per attempted visual extraction call; failures count as the 120s timeout", "sort": null } ] }, { "model": "openai/gpt-5.6-luna@low", "qualityScore": 86, "latencyP50Ms": 1764, "latencyP95Ms": 11000, "totalDurationMs": 36565, "costUsd": 0.0051741999999999995, "costPerCaseUsd": 0.00036958571428571425, "tokens": 51742, "casesAttempted": 14, "metrics": [ { "label": "accuracy", "value": "12/14 (86%)", "hint": "cases whose answers state every answer-key fact", "sort": 0.8571428571428571 }, { "label": "answered", "value": "14/14", "hint": "successful image-model calls; text-only models normally score zero", "sort": null }, { "label": "latency p50/p95", "value": "1.8s / 11.0s", "hint": "wall-clock per attempted visual extraction call; failures count as the 120s timeout", "sort": null } ] }, { "model": "x-ai/grok-4.5@low", "qualityScore": 86, "latencyP50Ms": 1872, "latencyP95Ms": 3028, "totalDurationMs": 28548, "costUsd": 0.053871999999999996, "costPerCaseUsd": 0.003848, "tokens": 26936, "casesAttempted": 14, "metrics": [ { "label": "accuracy", "value": "12/14 (86%)", "hint": "cases whose answers state every answer-key fact", "sort": 0.8571428571428571 }, { "label": "answered", "value": "14/14", "hint": "successful image-model calls; text-only models normally score zero", "sort": null }, { "label": "latency p50/p95", "value": "1.9s / 3.0s", "hint": "wall-clock per attempted visual extraction call; failures count as the 120s timeout", "sort": null } ] }, { "model": "anthropic/claude-sonnet-5@low", "qualityScore": 86, "latencyP50Ms": 3821, "latencyP95Ms": 6606, "totalDurationMs": 56317, "costUsd": 0.077042, "costPerCaseUsd": 0.005503, "tokens": 38521, "casesAttempted": 14, "metrics": [ { "label": "accuracy", "value": "12/14 (86%)", "hint": "cases whose answers state every answer-key fact", "sort": 0.8571428571428571 }, { "label": "answered", "value": "14/14", "hint": "successful image-model calls; text-only models normally score zero", "sort": null }, { "label": "latency p50/p95", "value": "3.8s / 6.6s", "hint": "wall-clock per attempted visual extraction call; failures count as the 120s timeout", "sort": null } ] }, { "model": "anthropic/claude-haiku-4.5@low", "qualityScore": 86, "latencyP50Ms": 5441, "latencyP95Ms": 31888, "totalDurationMs": 125020, "costUsd": 0.027091, "costPerCaseUsd": 0.0019350714285714286, "tokens": 27091, "casesAttempted": 14, "metrics": [ { "label": "accuracy", "value": "12/14 (86%)", "hint": "cases whose answers state every answer-key fact", "sort": 0.8571428571428571 }, { "label": "answered", "value": "14/14", "hint": "successful image-model calls; text-only models normally score zero", "sort": null }, { "label": "latency p50/p95", "value": "5.4s / 31.9s", "hint": "wall-clock per attempted visual extraction call; failures count as the 120s timeout", "sort": null } ] }, { "model": "anthropic/claude-opus-5@low", "qualityScore": 86, "latencyP50Ms": 7195, "latencyP95Ms": 12751, "totalDurationMs": 109361, "costUsd": 0.20809000000000002, "costPerCaseUsd": 0.01486357142857143, "tokens": 41618, "casesAttempted": 14, "metrics": [ { "label": "accuracy", "value": "12/14 (86%)", "hint": "cases whose answers state every answer-key fact", "sort": 0.8571428571428571 }, { "label": "answered", "value": "14/14", "hint": "successful image-model calls; text-only models normally score zero", "sort": null }, { "label": "latency p50/p95", "value": "7.2s / 12.8s", "hint": "wall-clock per attempted visual extraction call; failures count as the 120s timeout", "sort": null } ] }, { "model": "minimax/minimax-m3@low", "qualityScore": 79, "latencyP50Ms": 1655, "latencyP95Ms": 4792, "totalDurationMs": 28762, "costUsd": 0.0062442, "costPerCaseUsd": 0.00044601428571428573, "tokens": 20814, "casesAttempted": 14, "metrics": [ { "label": "accuracy", "value": "11/14 (79%)", "hint": "cases whose answers state every answer-key fact", "sort": 0.7857142857142857 }, { "label": "answered", "value": "14/14", "hint": "successful image-model calls; text-only models normally score zero", "sort": null }, { "label": "latency p50/p95", "value": "1.7s / 4.8s", "hint": "wall-clock per attempted visual extraction call; failures count as the 120s timeout", "sort": null } ] }, { "model": "moonshotai/kimi-k2.7-code@low", "qualityScore": 64, "latencyP50Ms": 5048, "latencyP95Ms": 120000, "totalDurationMs": 415922, "costUsd": 0.0186158, "costPerCaseUsd": 0.0013296999999999998, "tokens": 26594, "casesAttempted": 14, "metrics": [ { "label": "accuracy", "value": "9/14 (64%)", "hint": "cases whose answers state every answer-key fact", "sort": 0.6428571428571429 }, { "label": "answered", "value": "11/14", "hint": "successful image-model calls; text-only models normally score zero", "sort": null }, { "label": "latency p50/p95", "value": "5.0s / 120.0s", "hint": "wall-clock per attempted visual extraction call; failures count as the 120s timeout", "sort": null } ] }, { "model": "qwen/qwen3.8-max@low", "qualityScore": 64, "latencyP50Ms": 9437, "latencyP95Ms": 120000, "totalDurationMs": 571856, "costUsd": 0.040158, "costPerCaseUsd": 0.0028684285714285713, "tokens": 20079, "casesAttempted": 14, "metrics": [ { "label": "accuracy", "value": "9/14 (64%)", "hint": "cases whose answers state every answer-key fact", "sort": 0.6428571428571429 }, { "label": "answered", "value": "10/14", "hint": "successful image-model calls; text-only models normally score zero", "sort": null }, { "label": "latency p50/p95", "value": "9.4s / 120.0s", "hint": "wall-clock per attempted visual extraction call; failures count as the 120s timeout", "sort": null } ] }, { "model": "thinkingmachines/inkling@low", "qualityScore": 29, "latencyP50Ms": 120000, "latencyP95Ms": 120000, "totalDurationMs": 1099976, "costUsd": 0.0082802, "costPerCaseUsd": 0.0005914428571428571, "tokens": 8716, "casesAttempted": 14, "metrics": [ { "label": "accuracy", "value": "4/14 (29%)", "hint": "cases whose answers state every answer-key fact", "sort": 0.2857142857142857 }, { "label": "answered", "value": "5/14", "hint": "successful image-model calls; text-only models normally score zero", "sort": null }, { "label": "latency p50/p95", "value": "120.0s / 120.0s", "hint": "wall-clock per attempted visual extraction call; failures count as the 120s timeout", "sort": null } ] }, { "model": "bytedance-seed/seed-2.0-code@low", "qualityScore": 0, "latencyP50Ms": 120000, "latencyP95Ms": 120000, "totalDurationMs": 1680000, "costUsd": 0, "costPerCaseUsd": 0, "tokens": 0, "casesAttempted": 14, "metrics": [ { "label": "accuracy", "value": "0/14 (0%)", "hint": "cases whose answers state every answer-key fact", "sort": 0 }, { "label": "answered", "value": "0/14", "hint": "successful image-model calls; text-only models normally score zero", "sort": null }, { "label": "latency p50/p95", "value": "120.0s / 120.0s", "hint": "wall-clock per attempted visual extraction call; failures count as the 120s timeout", "sort": null } ] }, { "model": "deepseek/deepseek-v4-flash-0731@low", "qualityScore": 0, "latencyP50Ms": 120000, "latencyP95Ms": 120000, "totalDurationMs": 1680000, "costUsd": 0, "costPerCaseUsd": 0, "tokens": 0, "casesAttempted": 14, "metrics": [ { "label": "accuracy", "value": "0/14 (0%)", "hint": "cases whose answers state every answer-key fact", "sort": 0 }, { "label": "answered", "value": "0/14", "hint": "successful image-model calls; text-only models normally score zero", "sort": null }, { "label": "latency p50/p95", "value": "120.0s / 120.0s", "hint": "wall-clock per attempted visual extraction call; failures count as the 120s timeout", "sort": null } ] }, { "model": "deepseek/deepseek-v4-pro-0813@low", "qualityScore": 0, "latencyP50Ms": 120000, "latencyP95Ms": 120000, "totalDurationMs": 1680000, "costUsd": 0, "costPerCaseUsd": 0, "tokens": 0, "casesAttempted": 14, "metrics": [ { "label": "accuracy", "value": "0/14 (0%)", "hint": "cases whose answers state every answer-key fact", "sort": 0 }, { "label": "answered", "value": "0/14", "hint": "successful image-model calls; text-only models normally score zero", "sort": null }, { "label": "latency p50/p95", "value": "120.0s / 120.0s", "hint": "wall-clock per attempted visual extraction call; failures count as the 120s timeout", "sort": null } ] }, { "model": "deepseek/deepseek-v4-pro@low", "qualityScore": 0, "latencyP50Ms": 120000, "latencyP95Ms": 120000, "totalDurationMs": 1680000, "costUsd": 0, "costPerCaseUsd": 0, "tokens": 0, "casesAttempted": 14, "metrics": [ { "label": "accuracy", "value": "0/14 (0%)", "hint": "cases whose answers state every answer-key fact", "sort": 0 }, { "label": "answered", "value": "0/14", "hint": "successful image-model calls; text-only models normally score zero", "sort": null }, { "label": "latency p50/p95", "value": "120.0s / 120.0s", "hint": "wall-clock per attempted visual extraction call; failures count as the 120s timeout", "sort": null } ] }, { "model": "z-ai/glm-5.2@low", "qualityScore": 0, "latencyP50Ms": 120000, "latencyP95Ms": 120000, "totalDurationMs": 1680000, "costUsd": 0, "costPerCaseUsd": 0, "tokens": 0, "casesAttempted": 14, "metrics": [ { "label": "accuracy", "value": "0/14 (0%)", "hint": "cases whose answers state every answer-key fact", "sort": 0 }, { "label": "answered", "value": "0/14", "hint": "successful image-model calls; text-only models normally score zero", "sort": null }, { "label": "latency p50/p95", "value": "120.0s / 120.0s", "hint": "wall-clock per attempted visual extraction call; failures count as the 120s timeout", "sort": null } ] } ] }, { "id": "review", "name": "Review — self-review subagent", "description": "Runs Roomote's review prompt over a fixed benchmark of shipped diffs and measures how many known flaws it catches (a neutral meta-judge decides) and how often it blocks clean PRs.", "role": "codeReview", "cases": 84, "qualityMethod": "miss-weighted informedness", "updatedAt": "2026-08-18T15:09:32.810Z", "results": [ { "model": "anthropic/claude-fable-5@high", "qualityScore": 57, "latencyP50Ms": 20053, "latencyP95Ms": 63243, "totalDurationMs": 2083538, "costUsd": 5.01252, "costPerCaseUsd": 0.059672857142857146, "tokens": 501252, "casesAttempted": 84, "metrics": [ { "label": "catch rate", "value": "15/22 (68%)", "hint": "known flaws surfaced — higher is better", "sort": 0.6818181818181818 }, { "label": "false alarm", "value": "14/62 (23%)", "hint": "clean PRs blocked — lower is better", "sort": -0.22580645161290322 }, { "label": "issues per review", "value": "3.5", "hint": "average issues raised per successful review; high volume can inflate catch rate by shotgunning", "sort": null }, { "label": "issues per clean PR", "value": "2.7", "hint": "average issues raised on audited-clean diffs — every one is noise; lower is better", "sort": -2.7419354838709675 }, { "label": "latency p50/p95", "value": "20.1s / 63.2s", "hint": "wall-clock per attempted review call; failures count as the 15-minute timeout", "sort": null } ] }, { "model": "moonshotai/kimi-k3@high", "qualityScore": 54, "latencyP50Ms": 50844, "latencyP95Ms": 900000, "totalDurationMs": 13846309, "costUsd": 1.1157030000000001, "costPerCaseUsd": 0.013282178571428573, "tokens": 371901, "casesAttempted": 84, "metrics": [ { "label": "catch rate", "value": "15/22 (68%)", "hint": "known flaws surfaced — higher is better", "sort": 0.6818181818181818 }, { "label": "false alarm", "value": "18/62 (29%)", "hint": "clean PRs blocked — lower is better", "sort": -0.2903225806451613 }, { "label": "issues per review", "value": "3.8", "hint": "average issues raised per successful review; high volume can inflate catch rate by shotgunning", "sort": null }, { "label": "issues per clean PR", "value": "3.3", "hint": "average issues raised on audited-clean diffs — every one is noise; lower is better", "sort": -3.2545454545454544 }, { "label": "latency p50/p95", "value": "50.8s / 900.0s", "hint": "wall-clock per attempted review call; failures count as the 15-minute timeout", "sort": null } ] }, { "model": "z-ai/glm-5.2@high", "qualityScore": 51, "latencyP50Ms": 20320, "latencyP95Ms": 228704, "totalDurationMs": 4692153, "costUsd": 0.1348990272, "costPerCaseUsd": 0.0016059408, "tokens": 334571, "casesAttempted": 84, "metrics": [ { "label": "catch rate", "value": "12/22 (55%)", "hint": "known flaws surfaced — higher is better", "sort": 0.5454545454545454 }, { "label": "false alarm", "value": "4/62 (6%)", "hint": "clean PRs blocked — lower is better", "sort": -0.06451612903225806 }, { "label": "issues per review", "value": "2.4", "hint": "average issues raised per successful review; high volume can inflate catch rate by shotgunning", "sort": null }, { "label": "issues per clean PR", "value": "1.9", "hint": "average issues raised on audited-clean diffs — every one is noise; lower is better", "sort": -1.8852459016393444 }, { "label": "latency p50/p95", "value": "20.3s / 228.7s", "hint": "wall-clock per attempted review call; failures count as the 15-minute timeout", "sort": null } ] }, { "model": "anthropic/claude-opus-5@high", "qualityScore": 51, "latencyP50Ms": 27553, "latencyP95Ms": 78336, "totalDurationMs": 2991104, "costUsd": 2.921065, "costPerCaseUsd": 0.03477458333333333, "tokens": 584213, "casesAttempted": 84, "metrics": [ { "label": "catch rate", "value": "16/22 (73%)", "hint": "known flaws surfaced — higher is better", "sort": 0.7272727272727273 }, { "label": "false alarm", "value": "27/62 (44%)", "hint": "clean PRs blocked — lower is better", "sort": -0.43548387096774194 }, { "label": "issues per review", "value": "3.5", "hint": "average issues raised per successful review; high volume can inflate catch rate by shotgunning", "sort": null }, { "label": "issues per clean PR", "value": "2.7", "hint": "average issues raised on audited-clean diffs — every one is noise; lower is better", "sort": -2.6774193548387095 }, { "label": "latency p50/p95", "value": "27.6s / 78.3s", "hint": "wall-clock per attempted review call; failures count as the 15-minute timeout", "sort": null } ] }, { "model": "openai/gpt-5.6-terra@high", "qualityScore": 50, "latencyP50Ms": 9766, "latencyP95Ms": 45665, "totalDurationMs": 1388999, "costUsd": 0.312481, "costPerCaseUsd": 0.003720011904761905, "tokens": 312481, "casesAttempted": 84, "metrics": [ { "label": "catch rate", "value": "14/22 (64%)", "hint": "known flaws surfaced — higher is better", "sort": 0.6363636363636364 }, { "label": "false alarm", "value": "17/62 (27%)", "hint": "clean PRs blocked — lower is better", "sort": -0.27419354838709675 }, { "label": "issues per review", "value": "1.1", "hint": "average issues raised per successful review; high volume can inflate catch rate by shotgunning", "sort": null }, { "label": "issues per clean PR", "value": "0.7", "hint": "average issues raised on audited-clean diffs — every one is noise; lower is better", "sort": -0.6612903225806451 }, { "label": "latency p50/p95", "value": "9.8s / 45.7s", "hint": "wall-clock per attempted review call; failures count as the 15-minute timeout", "sort": null } ] }, { "model": "x-ai/grok-4.5@high", "qualityScore": 42, "latencyP50Ms": 30832, "latencyP95Ms": 126298, "totalDurationMs": 3482215, "costUsd": 0.85391, "costPerCaseUsd": 0.010165595238095237, "tokens": 426955, "casesAttempted": 84, "metrics": [ { "label": "catch rate", "value": "12/22 (55%)", "hint": "known flaws surfaced — higher is better", "sort": 0.5454545454545454 }, { "label": "false alarm", "value": "16/62 (26%)", "hint": "clean PRs blocked — lower is better", "sort": -0.25806451612903225 }, { "label": "issues per review", "value": "1.5", "hint": "average issues raised per successful review; high volume can inflate catch rate by shotgunning", "sort": null }, { "label": "issues per clean PR", "value": "0.9", "hint": "average issues raised on audited-clean diffs — every one is noise; lower is better", "sort": -0.9354838709677419 }, { "label": "latency p50/p95", "value": "30.8s / 126.3s", "hint": "wall-clock per attempted review call; failures count as the 15-minute timeout", "sort": null } ] }, { "model": "google/gemini-3.7-flash@high", "qualityScore": 40, "latencyP50Ms": 7331, "latencyP95Ms": 22362, "totalDurationMs": 786676, "costUsd": 0.1289325, "costPerCaseUsd": 0.0015349107142857144, "tokens": 343820, "casesAttempted": 84, "metrics": [ { "label": "catch rate", "value": "9/22 (41%)", "hint": "known flaws surfaced — higher is better", "sort": 0.4090909090909091 }, { "label": "false alarm", "value": "1/62 (2%)", "hint": "clean PRs blocked — lower is better", "sort": -0.016129032258064516 }, { "label": "issues per review", "value": "0.3", "hint": "average issues raised per successful review; high volume can inflate catch rate by shotgunning", "sort": null }, { "label": "issues per clean PR", "value": "0.0", "hint": "average issues raised on audited-clean diffs — every one is noise; lower is better", "sort": -0.016129032258064516 }, { "label": "latency p50/p95", "value": "7.3s / 22.4s", "hint": "wall-clock per attempted review call; failures count as the 15-minute timeout", "sort": null } ] }, { "model": "openai/gpt-5.6-luna@high", "qualityScore": 40, "latencyP50Ms": 24930, "latencyP95Ms": 75698, "totalDurationMs": 2638801, "costUsd": 0.0511574, "costPerCaseUsd": 0.0006090166666666666, "tokens": 511574, "casesAttempted": 84, "metrics": [ { "label": "catch rate", "value": "13/22 (59%)", "hint": "known flaws surfaced — higher is better", "sort": 0.5909090909090909 }, { "label": "false alarm", "value": "24/62 (39%)", "hint": "clean PRs blocked — lower is better", "sort": -0.3870967741935484 }, { "label": "issues per review", "value": "1.4", "hint": "average issues raised per successful review; high volume can inflate catch rate by shotgunning", "sort": null }, { "label": "issues per clean PR", "value": "0.9", "hint": "average issues raised on audited-clean diffs — every one is noise; lower is better", "sort": -0.8870967741935484 }, { "label": "latency p50/p95", "value": "24.9s / 75.7s", "hint": "wall-clock per attempted review call; failures count as the 15-minute timeout", "sort": null } ] }, { "model": "anthropic/claude-sonnet-5@high", "qualityScore": 39, "latencyP50Ms": 55038, "latencyP95Ms": 193348, "totalDurationMs": 5445414, "costUsd": 1.7488219999999999, "costPerCaseUsd": 0.02081930952380952, "tokens": 874411, "casesAttempted": 84, "metrics": [ { "label": "catch rate", "value": "13/22 (59%)", "hint": "known flaws surfaced — higher is better", "sort": 0.5909090909090909 }, { "label": "false alarm", "value": "25/62 (40%)", "hint": "clean PRs blocked — lower is better", "sort": -0.4032258064516129 }, { "label": "issues per review", "value": "2.8", "hint": "average issues raised per successful review; high volume can inflate catch rate by shotgunning", "sort": null }, { "label": "issues per clean PR", "value": "2.1", "hint": "average issues raised on audited-clean diffs — every one is noise; lower is better", "sort": -2.096774193548387 }, { "label": "latency p50/p95", "value": "55.0s / 193.3s", "hint": "wall-clock per attempted review call; failures count as the 15-minute timeout", "sort": null } ] }, { "model": "deepseek/deepseek-v4-flash-0731@high", "qualityScore": 38, "latencyP50Ms": 84631, "latencyP95Ms": 900000, "totalDurationMs": 28779594, "costUsd": 0.02786526, "costPerCaseUsd": 0.0003317292857142857, "tokens": 309614, "casesAttempted": 84, "metrics": [ { "label": "catch rate", "value": "9/22 (41%)", "hint": "known flaws surfaced — higher is better", "sort": 0.4090909090909091 }, { "label": "false alarm", "value": "4/62 (6%)", "hint": "clean PRs blocked — lower is better", "sort": -0.06451612903225806 }, { "label": "issues per review", "value": "1.9", "hint": "average issues raised per successful review; high volume can inflate catch rate by shotgunning", "sort": null }, { "label": "issues per clean PR", "value": "1.2", "hint": "average issues raised on audited-clean diffs — every one is noise; lower is better", "sort": -1.2432432432432432 }, { "label": "latency p50/p95", "value": "84.6s / 900.0s", "hint": "wall-clock per attempted review call; failures count as the 15-minute timeout", "sort": null } ] }, { "model": "qwen/qwen3.8-max@high", "qualityScore": 35, "latencyP50Ms": 170503, "latencyP95Ms": 355436, "totalDurationMs": 16608752, "costUsd": 1.738318, "costPerCaseUsd": 0.020694261904761906, "tokens": 869159, "casesAttempted": 84, "metrics": [ { "label": "catch rate", "value": "16/22 (73%)", "hint": "known flaws surfaced — higher is better", "sort": 0.7272727272727273 }, { "label": "false alarm", "value": "47/62 (76%)", "hint": "clean PRs blocked — lower is better", "sort": -0.7580645161290323 }, { "label": "issues per review", "value": "5.8", "hint": "average issues raised per successful review; high volume can inflate catch rate by shotgunning", "sort": null }, { "label": "issues per clean PR", "value": "5.5", "hint": "average issues raised on audited-clean diffs — every one is noise; lower is better", "sort": -5.5 }, { "label": "latency p50/p95", "value": "170.5s / 355.4s", "hint": "wall-clock per attempted review call; failures count as the 15-minute timeout", "sort": null } ] }, { "model": "openai/gpt-5.6-sol@high", "qualityScore": 34, "latencyP50Ms": 25096, "latencyP95Ms": 119386, "totalDurationMs": 3244411, "costUsd": 1.7577850000000002, "costPerCaseUsd": 0.020926011904761906, "tokens": 351557, "casesAttempted": 84, "metrics": [ { "label": "catch rate", "value": "12/22 (55%)", "hint": "known flaws surfaced — higher is better", "sort": 0.5454545454545454 }, { "label": "false alarm", "value": "25/62 (40%)", "hint": "clean PRs blocked — lower is better", "sort": -0.4032258064516129 }, { "label": "issues per review", "value": "1.3", "hint": "average issues raised per successful review; high volume can inflate catch rate by shotgunning", "sort": null }, { "label": "issues per clean PR", "value": "0.9", "hint": "average issues raised on audited-clean diffs — every one is noise; lower is better", "sort": -0.9354838709677419 }, { "label": "latency p50/p95", "value": "25.1s / 119.4s", "hint": "wall-clock per attempted review call; failures count as the 15-minute timeout", "sort": null } ] }, { "model": "anthropic/claude-haiku-4.5@high", "qualityScore": 34, "latencyP50Ms": 72094, "latencyP95Ms": 118127, "totalDurationMs": 6732736, "costUsd": 1.0259179999999999, "costPerCaseUsd": 0.012213309523809522, "tokens": 1025918, "casesAttempted": 84, "metrics": [ { "label": "catch rate", "value": "12/22 (55%)", "hint": "known flaws surfaced — higher is better", "sort": 0.5454545454545454 }, { "label": "false alarm", "value": "25/62 (40%)", "hint": "clean PRs blocked — lower is better", "sort": -0.4032258064516129 }, { "label": "issues per review", "value": "1.8", "hint": "average issues raised per successful review; high volume can inflate catch rate by shotgunning", "sort": null }, { "label": "issues per clean PR", "value": "1.0", "hint": "average issues raised on audited-clean diffs — every one is noise; lower is better", "sort": -1.0327868852459017 }, { "label": "latency p50/p95", "value": "72.1s / 118.1s", "hint": "wall-clock per attempted review call; failures count as the 15-minute timeout", "sort": null } ] }, { "model": "x-ai/grok-4.6@high", "qualityScore": 33, "latencyP50Ms": 32312, "latencyP95Ms": 160473, "totalDurationMs": 3952629, "costUsd": 0.972976, "costPerCaseUsd": 0.011583047619047618, "tokens": 486488, "casesAttempted": 84, "metrics": [ { "label": "catch rate", "value": "11/22 (50%)", "hint": "known flaws surfaced — higher is better", "sort": 0.5 }, { "label": "false alarm", "value": "21/62 (34%)", "hint": "clean PRs blocked — lower is better", "sort": -0.3387096774193548 }, { "label": "issues per review", "value": "1.2", "hint": "average issues raised per successful review; high volume can inflate catch rate by shotgunning", "sort": null }, { "label": "issues per clean PR", "value": "0.8", "hint": "average issues raised on audited-clean diffs — every one is noise; lower is better", "sort": -0.7741935483870968 }, { "label": "latency p50/p95", "value": "32.3s / 160.5s", "hint": "wall-clock per attempted review call; failures count as the 15-minute timeout", "sort": null } ] }, { "model": "google/gemini-3.6-flash@high", "qualityScore": 32, "latencyP50Ms": 12016, "latencyP95Ms": 66173, "totalDurationMs": 1652111, "costUsd": 0.8837565000000001, "costPerCaseUsd": 0.010520910714285715, "tokens": 589171, "casesAttempted": 84, "metrics": [ { "label": "catch rate", "value": "10/22 (45%)", "hint": "known flaws surfaced — higher is better", "sort": 0.45454545454545453 }, { "label": "false alarm", "value": "17/62 (27%)", "hint": "clean PRs blocked — lower is better", "sort": -0.27419354838709675 }, { "label": "issues per review", "value": "0.7", "hint": "average issues raised per successful review; high volume can inflate catch rate by shotgunning", "sort": null }, { "label": "issues per clean PR", "value": "0.4", "hint": "average issues raised on audited-clean diffs — every one is noise; lower is better", "sort": -0.41935483870967744 }, { "label": "latency p50/p95", "value": "12.0s / 66.2s", "hint": "wall-clock per attempted review call; failures count as the 15-minute timeout", "sort": null } ] }, { "model": "deepseek/deepseek-v4-pro-0813@high", "qualityScore": 27, "latencyP50Ms": 165329, "latencyP95Ms": 900000, "totalDurationMs": 28512696, "costUsd": 0.48196896, "costPerCaseUsd": 0.005737725714285715, "tokens": 730256, "casesAttempted": 84, "metrics": [ { "label": "catch rate", "value": "9/22 (41%)", "hint": "known flaws surfaced — higher is better", "sort": 0.4090909090909091 }, { "label": "false alarm", "value": "17/62 (27%)", "hint": "clean PRs blocked — lower is better", "sort": -0.27419354838709675 }, { "label": "issues per review", "value": "1.7", "hint": "average issues raised per successful review; high volume can inflate catch rate by shotgunning", "sort": null }, { "label": "issues per clean PR", "value": "1.2", "hint": "average issues raised on audited-clean diffs — every one is noise; lower is better", "sort": -1.1875 }, { "label": "latency p50/p95", "value": "165.3s / 900.0s", "hint": "wall-clock per attempted review call; failures count as the 15-minute timeout", "sort": null } ] }, { "model": "deepseek/deepseek-v4-pro@high", "qualityScore": 24, "latencyP50Ms": 37880, "latencyP95Ms": 900000, "totalDurationMs": 7587930, "costUsd": 0.163289865, "costPerCaseUsd": 0.0019439269642857143, "tokens": 375379, "casesAttempted": 84, "metrics": [ { "label": "catch rate", "value": "10/22 (45%)", "hint": "known flaws surfaced — higher is better", "sort": 0.45454545454545453 }, { "label": "false alarm", "value": "26/62 (42%)", "hint": "clean PRs blocked — lower is better", "sort": -0.41935483870967744 }, { "label": "issues per review", "value": "1.3", "hint": "average issues raised per successful review; high volume can inflate catch rate by shotgunning", "sort": null }, { "label": "issues per clean PR", "value": "0.9", "hint": "average issues raised on audited-clean diffs — every one is noise; lower is better", "sort": -0.864406779661017 }, { "label": "latency p50/p95", "value": "37.9s / 900.0s", "hint": "wall-clock per attempted review call; failures count as the 15-minute timeout", "sort": null } ] }, { "model": "moonshotai/kimi-k2.7-code@high", "qualityScore": 8, "latencyP50Ms": 288782, "latencyP95Ms": 900000, "totalDurationMs": 38835086, "costUsd": 0.37204299999999996, "costPerCaseUsd": 0.004429083333333333, "tokens": 531490, "casesAttempted": 84, "metrics": [ { "label": "catch rate", "value": "5/22 (23%)", "hint": "known flaws surfaced — higher is better", "sort": 0.22727272727272727 }, { "label": "false alarm", "value": "18/62 (29%)", "hint": "clean PRs blocked — lower is better", "sort": -0.2903225806451613 }, { "label": "issues per review", "value": "3.8", "hint": "average issues raised per successful review; high volume can inflate catch rate by shotgunning", "sort": null }, { "label": "issues per clean PR", "value": "3.5", "hint": "average issues raised on audited-clean diffs — every one is noise; lower is better", "sort": -3.473684210526316 }, { "label": "latency p50/p95", "value": "288.8s / 900.0s", "hint": "wall-clock per attempted review call; failures count as the 15-minute timeout", "sort": null } ] }, { "model": "minimax/minimax-m3@high", "qualityScore": 7, "latencyP50Ms": 998, "latencyP95Ms": 6398, "totalDurationMs": 140281, "costUsd": 0.0640314, "costPerCaseUsd": 0.0007622785714285715, "tokens": 213438, "casesAttempted": 84, "metrics": [ { "label": "catch rate", "value": "2/22 (9%)", "hint": "known flaws surfaced — higher is better", "sort": 0.09090909090909091 }, { "label": "false alarm", "value": "2/62 (3%)", "hint": "clean PRs blocked — lower is better", "sort": -0.03225806451612903 }, { "label": "issues per review", "value": "0.4", "hint": "average issues raised per successful review; high volume can inflate catch rate by shotgunning", "sort": null }, { "label": "issues per clean PR", "value": "0.2", "hint": "average issues raised on audited-clean diffs — every one is noise; lower is better", "sort": -0.22580645161290322 }, { "label": "latency p50/p95", "value": "1.0s / 6.4s", "hint": "wall-clock per attempted review call; failures count as the 15-minute timeout", "sort": null } ] }, { "model": "bytedance-seed/seed-2.0-code@high", "qualityScore": 0, "latencyP50Ms": 900000, "latencyP95Ms": 900000, "totalDurationMs": 75600000, "costUsd": 0, "costPerCaseUsd": 0, "tokens": 0, "casesAttempted": 84, "metrics": [ { "label": "catch rate", "value": "0/22 (0%)", "hint": "known flaws surfaced — higher is better", "sort": 0 }, { "label": "false alarm", "value": "0/62 (0%)", "hint": "clean PRs blocked — lower is better", "sort": 0 }, { "label": "issues per review", "value": "n/a", "hint": "average issues raised per successful review; high volume can inflate catch rate by shotgunning", "sort": null }, { "label": "issues per clean PR", "value": "n/a", "hint": "average issues raised on audited-clean diffs — every one is noise; lower is better", "sort": null }, { "label": "latency p50/p95", "value": "900.0s / 900.0s", "hint": "wall-clock per attempted review call; failures count as the 15-minute timeout", "sort": null } ] }, { "model": "thinkingmachines/inkling@high", "qualityScore": 0, "latencyP50Ms": 900000, "latencyP95Ms": 900000, "totalDurationMs": 74767374, "costUsd": 0.0041876, "costPerCaseUsd": 0.00004985238095238095, "tokens": 4408, "casesAttempted": 84, "metrics": [ { "label": "catch rate", "value": "0/22 (0%)", "hint": "known flaws surfaced — higher is better", "sort": 0 }, { "label": "false alarm", "value": "0/62 (0%)", "hint": "clean PRs blocked — lower is better", "sort": 0 }, { "label": "issues per review", "value": "2.0", "hint": "average issues raised per successful review; high volume can inflate catch rate by shotgunning", "sort": null }, { "label": "issues per clean PR", "value": "2.0", "hint": "average issues raised on audited-clean diffs — every one is noise; lower is better", "sort": -2 }, { "label": "latency p50/p95", "value": "900.0s / 900.0s", "hint": "wall-clock per attempted review call; failures count as the 15-minute timeout", "sort": null } ] } ] }, { "id": "explorer", "name": "Explorer — repository search", "description": "Runs opencode's real explore subagent (production prompt, glob/grep/read tools) against pinned repository checkouts and measures whether it finds the files the real fix modified, and how efficiently.", "role": "explore", "cases": 13, "qualityMethod": "recall-weighted F2", "updatedAt": "2026-08-19T14:42:53.281Z", "results": [ { "model": "openai/gpt-5.6-sol@low", "qualityScore": 90, "latencyP50Ms": 35797, "latencyP95Ms": 125628, "totalDurationMs": 606135, "costUsd": 0.47490675000000004, "costPerCaseUsd": 0.03653128846153846, "tokens": 1539, "casesAttempted": 13, "metrics": [ { "label": "file recall", "value": "92%", "hint": "of the pre-existing files the real fix modified, how many the explore agent found", "sort": 0.9230769230769231 }, { "label": "precision", "value": "90%", "hint": "of existing files the report named, how many the fix actually touched", "sort": 0.8999999999999999 }, { "label": "tool calls", "value": "21.5", "hint": "average tool invocations per successful search — lower is cheaper", "sort": -21.46153846153846 }, { "label": "answered", "value": "13/13", "hint": "searches that completed within the time budget", "sort": null }, { "label": "latency p50/p95", "value": "35.8s / 125.6s", "hint": "wall-clock per attempted search; failures count as the 4-minute timeout", "sort": null } ] }, { "model": "google/gemini-3.7-flash@low", "qualityScore": 89, "latencyP50Ms": 47979, "latencyP95Ms": 156352, "totalDurationMs": 793275, "costUsd": 0.04049872499999999, "costPerCaseUsd": 0.003115286538461538, "tokens": 48290, "casesAttempted": 13, "metrics": [ { "label": "file recall", "value": "90%", "hint": "of the pre-existing files the real fix modified, how many the explore agent found", "sort": 0.8974358974358974 }, { "label": "precision", "value": "90%", "hint": "of existing files the report named, how many the fix actually touched", "sort": 0.9001831501831501 }, { "label": "tool calls", "value": "12.8", "hint": "average tool invocations per successful search — lower is cheaper", "sort": -12.76923076923077 }, { "label": "answered", "value": "13/13", "hint": "searches that completed within the time budget", "sort": null }, { "label": "latency p50/p95", "value": "48.0s / 156.4s", "hint": "wall-clock per attempted search; failures count as the 4-minute timeout", "sort": null } ] }, { "model": "google/gemini-3.6-flash@low", "qualityScore": 85, "latencyP50Ms": 64981, "latencyP95Ms": 229496, "totalDurationMs": 1023415, "costUsd": 0.35686755000000003, "costPerCaseUsd": 0.027451350000000003, "tokens": 115423, "casesAttempted": 13, "metrics": [ { "label": "file recall", "value": "90%", "hint": "of the pre-existing files the real fix modified, how many the explore agent found", "sort": 0.8974358974358974 }, { "label": "precision", "value": "83%", "hint": "of existing files the report named, how many the fix actually touched", "sort": 0.8307692307692308 }, { "label": "tool calls", "value": "25.5", "hint": "average tool invocations per successful search — lower is cheaper", "sort": -25.53846153846154 }, { "label": "answered", "value": "13/13", "hint": "searches that completed within the time budget", "sort": null }, { "label": "latency p50/p95", "value": "65.0s / 229.5s", "hint": "wall-clock per attempted search; failures count as the 4-minute timeout", "sort": null } ] }, { "model": "openai/gpt-5.6-terra@low", "qualityScore": 84, "latencyP50Ms": 18846, "latencyP95Ms": 56637, "totalDurationMs": 288106, "costUsd": 0.09048460000000001, "costPerCaseUsd": 0.006960353846153847, "tokens": 2634, "casesAttempted": 13, "metrics": [ { "label": "file recall", "value": "87%", "hint": "of the pre-existing files the real fix modified, how many the explore agent found", "sort": 0.8717948717948718 }, { "label": "precision", "value": "80%", "hint": "of existing files the report named, how many the fix actually touched", "sort": 0.8012820512820513 }, { "label": "tool calls", "value": "16.5", "hint": "average tool invocations per successful search — lower is cheaper", "sort": -16.46153846153846 }, { "label": "answered", "value": "13/13", "hint": "searches that completed within the time budget", "sort": null }, { "label": "latency p50/p95", "value": "18.8s / 56.6s", "hint": "wall-clock per attempted search; failures count as the 4-minute timeout", "sort": null } ] }, { "model": "openai/gpt-5.6-luna@low", "qualityScore": 83, "latencyP50Ms": 21278, "latencyP95Ms": 35274, "totalDurationMs": 267476, "costUsd": 0.008791509999999999, "costPerCaseUsd": 0.0006762699999999999, "tokens": 2019, "casesAttempted": 13, "metrics": [ { "label": "file recall", "value": "90%", "hint": "of the pre-existing files the real fix modified, how many the explore agent found", "sort": 0.8974358974358974 }, { "label": "precision", "value": "76%", "hint": "of existing files the report named, how many the fix actually touched", "sort": 0.7589743589743588 }, { "label": "tool calls", "value": "17.5", "hint": "average tool invocations per successful search — lower is cheaper", "sort": -17.53846153846154 }, { "label": "answered", "value": "13/13", "hint": "searches that completed within the time budget", "sort": null }, { "label": "latency p50/p95", "value": "21.3s / 35.3s", "hint": "wall-clock per attempted search; failures count as the 4-minute timeout", "sort": null } ] }, { "model": "x-ai/grok-4.6@low", "qualityScore": 81, "latencyP50Ms": 24226, "latencyP95Ms": 90633, "totalDurationMs": 455871, "costUsd": 0.263412, "costPerCaseUsd": 0.020262461538461538, "tokens": 44351, "casesAttempted": 13, "metrics": [ { "label": "file recall", "value": "92%", "hint": "of the pre-existing files the real fix modified, how many the explore agent found", "sort": 0.9230769230769231 }, { "label": "precision", "value": "70%", "hint": "of existing files the report named, how many the fix actually touched", "sort": 0.6961538461538461 }, { "label": "tool calls", "value": "15.4", "hint": "average tool invocations per successful search — lower is cheaper", "sort": -15.384615384615385 }, { "label": "answered", "value": "13/13", "hint": "searches that completed within the time budget", "sort": null }, { "label": "latency p50/p95", "value": "24.2s / 90.6s", "hint": "wall-clock per attempted search; failures count as the 4-minute timeout", "sort": null } ] }, { "model": "anthropic/claude-sonnet-5@low", "qualityScore": 77, "latencyP50Ms": 25876, "latencyP95Ms": 164858, "totalDurationMs": 530426, "costUsd": 0.2670901, "costPerCaseUsd": 0.020545392307692308, "tokens": 9125, "casesAttempted": 13, "metrics": [ { "label": "file recall", "value": "92%", "hint": "of the pre-existing files the real fix modified, how many the explore agent found", "sort": 0.9230769230769231 }, { "label": "precision", "value": "61%", "hint": "of existing files the report named, how many the fix actually touched", "sort": 0.6052197802197802 }, { "label": "tool calls", "value": "11.0", "hint": "average tool invocations per successful search — lower is cheaper", "sort": -11 }, { "label": "answered", "value": "13/13", "hint": "searches that completed within the time budget", "sort": null }, { "label": "latency p50/p95", "value": "25.9s / 164.9s", "hint": "wall-clock per attempted search; failures count as the 4-minute timeout", "sort": null } ] }, { "model": "anthropic/claude-fable-5@low", "qualityScore": 73, "latencyP50Ms": 24924, "latencyP95Ms": 107346, "totalDurationMs": 489681, "costUsd": 1.451734, "costPerCaseUsd": 0.11167184615384616, "tokens": 11796, "casesAttempted": 13, "metrics": [ { "label": "file recall", "value": "92%", "hint": "of the pre-existing files the real fix modified, how many the explore agent found", "sort": 0.9230769230769231 }, { "label": "precision", "value": "45%", "hint": "of existing files the report named, how many the fix actually touched", "sort": 0.44667832167832167 }, { "label": "tool calls", "value": "8.0", "hint": "average tool invocations per successful search — lower is cheaper", "sort": -8 }, { "label": "answered", "value": "13/13", "hint": "searches that completed within the time budget", "sort": null }, { "label": "latency p50/p95", "value": "24.9s / 107.3s", "hint": "wall-clock per attempted search; failures count as the 4-minute timeout", "sort": null } ] }, { "model": "thinkingmachines/inkling@low", "qualityScore": 72, "latencyP50Ms": 15336, "latencyP95Ms": 45428, "totalDurationMs": 238118, "costUsd": 0.06998634999999999, "costPerCaseUsd": 0.005383565384615384, "tokens": 35674, "casesAttempted": 13, "metrics": [ { "label": "file recall", "value": "74%", "hint": "of the pre-existing files the real fix modified, how many the explore agent found", "sort": 0.7435897435897437 }, { "label": "precision", "value": "72%", "hint": "of existing files the report named, how many the fix actually touched", "sort": 0.717948717948718 }, { "label": "tool calls", "value": "6.1", "hint": "average tool invocations per successful search — lower is cheaper", "sort": -6.076923076923077 }, { "label": "answered", "value": "13/13", "hint": "searches that completed within the time budget", "sort": null }, { "label": "latency p50/p95", "value": "15.3s / 45.4s", "hint": "wall-clock per attempted search; failures count as the 4-minute timeout", "sort": null } ] }, { "model": "x-ai/grok-4.5@low", "qualityScore": 71, "latencyP50Ms": 32390, "latencyP95Ms": 76614, "totalDurationMs": 456322, "costUsd": 0.16204639999999998, "costPerCaseUsd": 0.01246510769230769, "tokens": 35096, "casesAttempted": 13, "metrics": [ { "label": "file recall", "value": "90%", "hint": "of the pre-existing files the real fix modified, how many the explore agent found", "sort": 0.8974358974358974 }, { "label": "precision", "value": "52%", "hint": "of existing files the report named, how many the fix actually touched", "sort": 0.5245421245421246 }, { "label": "tool calls", "value": "16.3", "hint": "average tool invocations per successful search — lower is cheaper", "sort": -16.307692307692307 }, { "label": "answered", "value": "13/13", "hint": "searches that completed within the time budget", "sort": null }, { "label": "latency p50/p95", "value": "32.4s / 76.6s", "hint": "wall-clock per attempted search; failures count as the 4-minute timeout", "sort": null } ] }, { "model": "z-ai/glm-5.2@low", "qualityScore": 70, "latencyP50Ms": 32515, "latencyP95Ms": 80476, "totalDurationMs": 470247, "costUsd": 0.03754311144, "costPerCaseUsd": 0.0028879316492307692, "tokens": 32729, "casesAttempted": 13, "metrics": [ { "label": "file recall", "value": "90%", "hint": "of the pre-existing files the real fix modified, how many the explore agent found", "sort": 0.8974358974358974 }, { "label": "precision", "value": "49%", "hint": "of existing files the report named, how many the fix actually touched", "sort": 0.4883782883782884 }, { "label": "tool calls", "value": "15.8", "hint": "average tool invocations per successful search — lower is cheaper", "sort": -15.76923076923077 }, { "label": "answered", "value": "13/13", "hint": "searches that completed within the time budget", "sort": null }, { "label": "latency p50/p95", "value": "32.5s / 80.5s", "hint": "wall-clock per attempted search; failures count as the 4-minute timeout", "sort": null } ] }, { "model": "deepseek/deepseek-v4-flash-0731@low", "qualityScore": 69, "latencyP50Ms": 25359, "latencyP95Ms": 106825, "totalDurationMs": 536631, "costUsd": 0.008496756000000001, "costPerCaseUsd": 0.0006535966153846154, "tokens": 25473, "casesAttempted": 13, "metrics": [ { "label": "file recall", "value": "90%", "hint": "of the pre-existing files the real fix modified, how many the explore agent found", "sort": 0.8974358974358974 }, { "label": "precision", "value": "50%", "hint": "of existing files the report named, how many the fix actually touched", "sort": 0.49879336349924586 }, { "label": "tool calls", "value": "13.2", "hint": "average tool invocations per successful search — lower is cheaper", "sort": -13.23076923076923 }, { "label": "answered", "value": "13/13", "hint": "searches that completed within the time budget", "sort": null }, { "label": "latency p50/p95", "value": "25.4s / 106.8s", "hint": "wall-clock per attempted search; failures count as the 4-minute timeout", "sort": null } ] }, { "model": "deepseek/deepseek-v4-pro-0813@low", "qualityScore": 69, "latencyP50Ms": 46541, "latencyP95Ms": 126864, "totalDurationMs": 802134, "costUsd": 0.051963867999999996, "costPerCaseUsd": 0.003997220615384615, "tokens": 40281, "casesAttempted": 13, "metrics": [ { "label": "file recall", "value": "90%", "hint": "of the pre-existing files the real fix modified, how many the explore agent found", "sort": 0.8974358974358974 }, { "label": "precision", "value": "50%", "hint": "of existing files the report named, how many the fix actually touched", "sort": 0.502014652014652 }, { "label": "tool calls", "value": "12.2", "hint": "average tool invocations per successful search — lower is cheaper", "sort": -12.153846153846153 }, { "label": "answered", "value": "13/13", "hint": "searches that completed within the time budget", "sort": null }, { "label": "latency p50/p95", "value": "46.5s / 126.9s", "hint": "wall-clock per attempted search; failures count as the 4-minute timeout", "sort": null } ] }, { "model": "anthropic/claude-opus-5@low", "qualityScore": 62, "latencyP50Ms": 29115, "latencyP95Ms": 87490, "totalDurationMs": 521180, "costUsd": 0.6518697499999999, "costPerCaseUsd": 0.050143826923076916, "tokens": 11676, "casesAttempted": 13, "metrics": [ { "label": "file recall", "value": "92%", "hint": "of the pre-existing files the real fix modified, how many the explore agent found", "sort": 0.9230769230769231 }, { "label": "precision", "value": "37%", "hint": "of existing files the report named, how many the fix actually touched", "sort": 0.3661172161172161 }, { "label": "tool calls", "value": "10.8", "hint": "average tool invocations per successful search — lower is cheaper", "sort": -10.76923076923077 }, { "label": "answered", "value": "13/13", "hint": "searches that completed within the time budget", "sort": null }, { "label": "latency p50/p95", "value": "29.1s / 87.5s", "hint": "wall-clock per attempted search; failures count as the 4-minute timeout", "sort": null } ] }, { "model": "anthropic/claude-haiku-4.5@low", "qualityScore": 60, "latencyP50Ms": 44748, "latencyP95Ms": 240000, "totalDurationMs": 701644, "costUsd": 0.07476239999999999, "costPerCaseUsd": 0.005750953846153845, "tokens": 5460, "casesAttempted": 13, "metrics": [ { "label": "file recall", "value": "77%", "hint": "of the pre-existing files the real fix modified, how many the explore agent found", "sort": 0.7692307692307693 }, { "label": "precision", "value": "47%", "hint": "of existing files the report named, how many the fix actually touched", "sort": 0.46880341880341886 }, { "label": "tool calls", "value": "23.3", "hint": "average tool invocations per successful search — lower is cheaper", "sort": -23.25 }, { "label": "answered", "value": "12/13", "hint": "searches that completed within the time budget", "sort": null }, { "label": "latency p50/p95", "value": "44.7s / 240.0s", "hint": "wall-clock per attempted search; failures count as the 4-minute timeout", "sort": null } ] }, { "model": "moonshotai/kimi-k3@low", "qualityScore": 55, "latencyP50Ms": 128055, "latencyP95Ms": 240000, "totalDurationMs": 1829764, "costUsd": 0.16537800000000002, "costPerCaseUsd": 0.012721384615384617, "tokens": 11173, "casesAttempted": 13, "metrics": [ { "label": "file recall", "value": "72%", "hint": "of the pre-existing files the real fix modified, how many the explore agent found", "sort": 0.717948717948718 }, { "label": "precision", "value": "37%", "hint": "of existing files the report named, how many the fix actually touched", "sort": 0.3743589743589744 }, { "label": "tool calls", "value": "15.1", "hint": "average tool invocations per successful search — lower is cheaper", "sort": -15.1 }, { "label": "answered", "value": "10/13", "hint": "searches that completed within the time budget", "sort": null }, { "label": "latency p50/p95", "value": "128.1s / 240.0s", "hint": "wall-clock per attempted search; failures count as the 4-minute timeout", "sort": null } ] }, { "model": "qwen/qwen3.8-max@low", "qualityScore": 53, "latencyP50Ms": 96570, "latencyP95Ms": 240000, "totalDurationMs": 1526760, "costUsd": 0.126766, "costPerCaseUsd": 0.009751230769230769, "tokens": 20397, "casesAttempted": 13, "metrics": [ { "label": "file recall", "value": "72%", "hint": "of the pre-existing files the real fix modified, how many the explore agent found", "sort": 0.717948717948718 }, { "label": "precision", "value": "35%", "hint": "of existing files the report named, how many the fix actually touched", "sort": 0.35201465201465204 }, { "label": "tool calls", "value": "33.3", "hint": "average tool invocations per successful search — lower is cheaper", "sort": -33.3 }, { "label": "answered", "value": "10/13", "hint": "searches that completed within the time budget", "sort": null }, { "label": "latency p50/p95", "value": "96.6s / 240.0s", "hint": "wall-clock per attempted search; failures count as the 4-minute timeout", "sort": null } ] }, { "model": "moonshotai/kimi-k2.7-code@low", "qualityScore": 53, "latencyP50Ms": 180895, "latencyP95Ms": 240000, "totalDurationMs": 2024686, "costUsd": 0.0608279, "costPerCaseUsd": 0.00467906923076923, "tokens": 22923, "casesAttempted": 13, "metrics": [ { "label": "file recall", "value": "64%", "hint": "of the pre-existing files the real fix modified, how many the explore agent found", "sort": 0.6410256410256411 }, { "label": "precision", "value": "42%", "hint": "of existing files the report named, how many the fix actually touched", "sort": 0.4224358974358975 }, { "label": "tool calls", "value": "23.0", "hint": "average tool invocations per successful search — lower is cheaper", "sort": -23 }, { "label": "answered", "value": "9/13", "hint": "searches that completed within the time budget", "sort": null }, { "label": "latency p50/p95", "value": "180.9s / 240.0s", "hint": "wall-clock per attempted search; failures count as the 4-minute timeout", "sort": null } ] }, { "model": "deepseek/deepseek-v4-pro@low", "qualityScore": 52, "latencyP50Ms": 78804, "latencyP95Ms": 183655, "totalDurationMs": 1138233, "costUsd": 0.044462626, "costPerCaseUsd": 0.003420202, "tokens": 80009, "casesAttempted": 13, "metrics": [ { "label": "file recall", "value": "74%", "hint": "of the pre-existing files the real fix modified, how many the explore agent found", "sort": 0.7435897435897435 }, { "label": "precision", "value": "39%", "hint": "of existing files the report named, how many the fix actually touched", "sort": 0.38670545141133383 }, { "label": "tool calls", "value": "29.4", "hint": "average tool invocations per successful search — lower is cheaper", "sort": -29.384615384615383 }, { "label": "answered", "value": "13/13", "hint": "searches that completed within the time budget", "sort": null }, { "label": "latency p50/p95", "value": "78.8s / 183.7s", "hint": "wall-clock per attempted search; failures count as the 4-minute timeout", "sort": null } ] }, { "model": "minimax/minimax-m3@low", "qualityScore": 47, "latencyP50Ms": 148462, "latencyP95Ms": 240000, "totalDurationMs": 2165198, "costUsd": 0.021845999999999997, "costPerCaseUsd": 0.0016804615384615382, "tokens": 20323, "casesAttempted": 13, "metrics": [ { "label": "file recall", "value": "56%", "hint": "of the pre-existing files the real fix modified, how many the explore agent found", "sort": 0.5641025641025641 }, { "label": "precision", "value": "40%", "hint": "of existing files the report named, how many the fix actually touched", "sort": 0.3961538461538462 }, { "label": "tool calls", "value": "26.0", "hint": "average tool invocations per successful search — lower is cheaper", "sort": -26 }, { "label": "answered", "value": "8/13", "hint": "searches that completed within the time budget", "sort": null }, { "label": "latency p50/p95", "value": "148.5s / 240.0s", "hint": "wall-clock per attempted search; failures count as the 4-minute timeout", "sort": null } ] }, { "model": "poolside/laguna-s-2.1", "qualityScore": 41, "latencyP50Ms": 167163, "latencyP95Ms": 240000, "totalDurationMs": 2123295, "costUsd": 0.004638852, "costPerCaseUsd": 0.0003568347692307692, "tokens": 18193, "casesAttempted": 13, "metrics": [ { "label": "file recall", "value": "51%", "hint": "of the pre-existing files the real fix modified, how many the explore agent found", "sort": 0.5128205128205129 }, { "label": "precision", "value": "28%", "hint": "of existing files the report named, how many the fix actually touched", "sort": 0.27820512820512816 }, { "label": "tool calls", "value": "21.1", "hint": "average tool invocations per successful search — lower is cheaper", "sort": -21.142857142857142 }, { "label": "answered", "value": "7/13", "hint": "searches that completed within the time budget", "sort": null }, { "label": "latency p50/p95", "value": "167.2s / 240.0s", "hint": "wall-clock per attempted search; failures count as the 4-minute timeout", "sort": null } ] } ] }, { "id": "advisor", "name": "Advisor", "description": "Runs the advisor prompt over consultations from its three production modes (upfront plans, stuck agents, user challenges) and grades free-form guidance against ground-truth answer keys; challenge stances are graded mechanically.", "role": "advisor", "cases": 32, "qualityMethod": "mode-averaged plan score", "updatedAt": "2026-08-17T20:29:05.753Z", "results": [ { "model": "anthropic/claude-opus-5@high", "qualityScore": 83, "latencyP50Ms": 75420, "latencyP95Ms": 129482, "totalDurationMs": 2600037, "costUsd": 0.92993, "costPerCaseUsd": 0.0290603125, "tokens": 185986, "casesAttempted": 32, "metrics": [ { "label": "plan mode", "value": "94%", "hint": "answer-key coverage on upfront plan requests", "sort": 0.9351851851851853 }, { "label": "unstick mode", "value": "80%", "hint": "answer-key coverage on stuck-agent consultations", "sort": 0.8030303030303029 }, { "label": "challenge mode", "value": "75%", "hint": "stance-gated grounding on user challenges", "sort": 0.75 }, { "label": "stance accuracy", "value": "9/12", "hint": "concede when the user is right, hold when the user is wrong", "sort": 0.75 }, { "label": "file recall", "value": "75%", "hint": "share of the real fix's files named in upfront plans — diagnostic, not part of the score", "sort": 0.7518518518518519 }, { "label": "answered", "value": "32/32", "hint": "successful structured advisor calls", "sort": null }, { "label": "latency p50/p95", "value": "75.4s / 129.5s", "hint": "wall-clock per attempted advisor call; failures count as the 5-minute timeout", "sort": null } ] }, { "model": "x-ai/grok-4.5@high", "qualityScore": 80, "latencyP50Ms": 49380, "latencyP95Ms": 107406, "totalDurationMs": 1768550, "costUsd": 0.186082, "costPerCaseUsd": 0.0058150625, "tokens": 93041, "casesAttempted": 32, "metrics": [ { "label": "plan mode", "value": "87%", "hint": "answer-key coverage on upfront plan requests", "sort": 0.8666666666666667 }, { "label": "unstick mode", "value": "77%", "hint": "answer-key coverage on stuck-agent consultations", "sort": 0.7727272727272727 }, { "label": "challenge mode", "value": "75%", "hint": "stance-gated grounding on user challenges", "sort": 0.75 }, { "label": "stance accuracy", "value": "9/12", "hint": "concede when the user is right, hold when the user is wrong", "sort": 0.75 }, { "label": "file recall", "value": "61%", "hint": "share of the real fix's files named in upfront plans — diagnostic, not part of the score", "sort": 0.6129629629629629 }, { "label": "answered", "value": "32/32", "hint": "successful structured advisor calls", "sort": null }, { "label": "latency p50/p95", "value": "49.4s / 107.4s", "hint": "wall-clock per attempted advisor call; failures count as the 5-minute timeout", "sort": null } ] }, { "model": "openai/gpt-5.6-sol@high", "qualityScore": 80, "latencyP50Ms": 61198, "latencyP95Ms": 181075, "totalDurationMs": 2532626, "costUsd": 0.5925450000000001, "costPerCaseUsd": 0.018517031250000003, "tokens": 118509, "casesAttempted": 32, "metrics": [ { "label": "plan mode", "value": "97%", "hint": "answer-key coverage on upfront plan requests", "sort": 0.9722222222222222 }, { "label": "unstick mode", "value": "71%", "hint": "answer-key coverage on stuck-agent consultations", "sort": 0.712121212121212 }, { "label": "challenge mode", "value": "72%", "hint": "stance-gated grounding on user challenges", "sort": 0.7222222222222222 }, { "label": "stance accuracy", "value": "9/12", "hint": "concede when the user is right, hold when the user is wrong", "sort": 0.75 }, { "label": "file recall", "value": "38%", "hint": "share of the real fix's files named in upfront plans — diagnostic, not part of the score", "sort": 0.37777777777777777 }, { "label": "answered", "value": "32/32", "hint": "successful structured advisor calls", "sort": null }, { "label": "latency p50/p95", "value": "61.2s / 181.1s", "hint": "wall-clock per attempted advisor call; failures count as the 5-minute timeout", "sort": null } ] }, { "model": "openai/gpt-5.6-terra@high", "qualityScore": 78, "latencyP50Ms": 42162, "latencyP95Ms": 83942, "totalDurationMs": 1583205, "costUsd": 0.09654399999999999, "costPerCaseUsd": 0.0030169999999999997, "tokens": 96544, "casesAttempted": 32, "metrics": [ { "label": "plan mode", "value": "97%", "hint": "answer-key coverage on upfront plan requests", "sort": 0.9722222222222222 }, { "label": "unstick mode", "value": "70%", "hint": "answer-key coverage on stuck-agent consultations", "sort": 0.6969696969696969 }, { "label": "challenge mode", "value": "67%", "hint": "stance-gated grounding on user challenges", "sort": 0.6666666666666666 }, { "label": "stance accuracy", "value": "8/12", "hint": "concede when the user is right, hold when the user is wrong", "sort": 0.6666666666666666 }, { "label": "file recall", "value": "48%", "hint": "share of the real fix's files named in upfront plans — diagnostic, not part of the score", "sort": 0.48148148148148145 }, { "label": "answered", "value": "32/32", "hint": "successful structured advisor calls", "sort": null }, { "label": "latency p50/p95", "value": "42.2s / 83.9s", "hint": "wall-clock per attempted advisor call; failures count as the 5-minute timeout", "sort": null } ] }, { "model": "openai/gpt-5.6-luna@high", "qualityScore": 75, "latencyP50Ms": 42551, "latencyP95Ms": 71256, "totalDurationMs": 1430093, "costUsd": 0.010007499999999999, "costPerCaseUsd": 0.00031273437499999997, "tokens": 100075, "casesAttempted": 32, "metrics": [ { "label": "plan mode", "value": "97%", "hint": "answer-key coverage on upfront plan requests", "sort": 0.9722222222222222 }, { "label": "unstick mode", "value": "62%", "hint": "answer-key coverage on stuck-agent consultations", "sort": 0.6212121212121212 }, { "label": "challenge mode", "value": "67%", "hint": "stance-gated grounding on user challenges", "sort": 0.6666666666666666 }, { "label": "stance accuracy", "value": "8/12", "hint": "concede when the user is right, hold when the user is wrong", "sort": 0.6666666666666666 }, { "label": "file recall", "value": "49%", "hint": "share of the real fix's files named in upfront plans — diagnostic, not part of the score", "sort": 0.49259259259259264 }, { "label": "answered", "value": "32/32", "hint": "successful structured advisor calls", "sort": null }, { "label": "latency p50/p95", "value": "42.6s / 71.3s", "hint": "wall-clock per attempted advisor call; failures count as the 5-minute timeout", "sort": null } ] }, { "model": "anthropic/claude-fable-5@high", "qualityScore": 75, "latencyP50Ms": 51728, "latencyP95Ms": 88677, "totalDurationMs": 1775903, "costUsd": 1.3774300000000002, "costPerCaseUsd": 0.043044687500000005, "tokens": 137743, "casesAttempted": 32, "metrics": [ { "label": "plan mode", "value": "94%", "hint": "answer-key coverage on upfront plan requests", "sort": 0.9444444444444444 }, { "label": "unstick mode", "value": "64%", "hint": "answer-key coverage on stuck-agent consultations", "sort": 0.6363636363636362 }, { "label": "challenge mode", "value": "67%", "hint": "stance-gated grounding on user challenges", "sort": 0.6666666666666666 }, { "label": "stance accuracy", "value": "9/12", "hint": "concede when the user is right, hold when the user is wrong", "sort": 0.75 }, { "label": "file recall", "value": "65%", "hint": "share of the real fix's files named in upfront plans — diagnostic, not part of the score", "sort": 0.6462962962962964 }, { "label": "answered", "value": "32/32", "hint": "successful structured advisor calls", "sort": null }, { "label": "latency p50/p95", "value": "51.7s / 88.7s", "hint": "wall-clock per attempted advisor call; failures count as the 5-minute timeout", "sort": null } ] }, { "model": "google/gemini-3.7-flash@high", "qualityScore": 71, "latencyP50Ms": 43121, "latencyP95Ms": 85097, "totalDurationMs": 1569968, "costUsd": 0.0472725, "costPerCaseUsd": 0.001477265625, "tokens": 126060, "casesAttempted": 32, "metrics": [ { "label": "plan mode", "value": "94%", "hint": "answer-key coverage on upfront plan requests", "sort": 0.9444444444444444 }, { "label": "unstick mode", "value": "45%", "hint": "answer-key coverage on stuck-agent consultations", "sort": 0.45454545454545453 }, { "label": "challenge mode", "value": "72%", "hint": "stance-gated grounding on user challenges", "sort": 0.7222222222222223 }, { "label": "stance accuracy", "value": "9/12", "hint": "concede when the user is right, hold when the user is wrong", "sort": 0.75 }, { "label": "file recall", "value": "61%", "hint": "share of the real fix's files named in upfront plans — diagnostic, not part of the score", "sort": 0.6092592592592593 }, { "label": "answered", "value": "32/32", "hint": "successful structured advisor calls", "sort": null }, { "label": "latency p50/p95", "value": "43.1s / 85.1s", "hint": "wall-clock per attempted advisor call; failures count as the 5-minute timeout", "sort": null } ] }, { "model": "x-ai/grok-4.6@high", "qualityScore": 67, "latencyP50Ms": 55166, "latencyP95Ms": 166722, "totalDurationMs": 2124135, "costUsd": 0.23501999999999998, "costPerCaseUsd": 0.007344374999999999, "tokens": 117510, "casesAttempted": 32, "metrics": [ { "label": "plan mode", "value": "78%", "hint": "answer-key coverage on upfront plan requests", "sort": 0.7777777777777778 }, { "label": "unstick mode", "value": "56%", "hint": "answer-key coverage on stuck-agent consultations", "sort": 0.5606060606060606 }, { "label": "challenge mode", "value": "67%", "hint": "stance-gated grounding on user challenges", "sort": 0.6666666666666666 }, { "label": "stance accuracy", "value": "8/12", "hint": "concede when the user is right, hold when the user is wrong", "sort": 0.6666666666666666 }, { "label": "file recall", "value": "53%", "hint": "share of the real fix's files named in upfront plans — diagnostic, not part of the score", "sort": 0.5296296296296297 }, { "label": "answered", "value": "32/32", "hint": "successful structured advisor calls", "sort": null }, { "label": "latency p50/p95", "value": "55.2s / 166.7s", "hint": "wall-clock per attempted advisor call; failures count as the 5-minute timeout", "sort": null } ] }, { "model": "google/gemini-3.6-flash@high", "qualityScore": 66, "latencyP50Ms": 39452, "latencyP95Ms": 61638, "totalDurationMs": 1293696, "costUsd": 0.1851195, "costPerCaseUsd": 0.005784984375, "tokens": 123413, "casesAttempted": 32, "metrics": [ { "label": "plan mode", "value": "92%", "hint": "answer-key coverage on upfront plan requests", "sort": 0.9166666666666666 }, { "label": "unstick mode", "value": "42%", "hint": "answer-key coverage on stuck-agent consultations", "sort": 0.4242424242424242 }, { "label": "challenge mode", "value": "64%", "hint": "stance-gated grounding on user challenges", "sort": 0.638888888888889 }, { "label": "stance accuracy", "value": "8/12", "hint": "concede when the user is right, hold when the user is wrong", "sort": 0.6666666666666666 }, { "label": "file recall", "value": "78%", "hint": "share of the real fix's files named in upfront plans — diagnostic, not part of the score", "sort": 0.7796296296296297 }, { "label": "answered", "value": "32/32", "hint": "successful structured advisor calls", "sort": null }, { "label": "latency p50/p95", "value": "39.5s / 61.6s", "hint": "wall-clock per attempted advisor call; failures count as the 5-minute timeout", "sort": null } ] }, { "model": "anthropic/claude-sonnet-5@high", "qualityScore": 65, "latencyP50Ms": 64325, "latencyP95Ms": 180491, "totalDurationMs": 2442385, "costUsd": 0.39866799999999997, "costPerCaseUsd": 0.012458374999999999, "tokens": 199334, "casesAttempted": 32, "metrics": [ { "label": "plan mode", "value": "92%", "hint": "answer-key coverage on upfront plan requests", "sort": 0.9222222222222223 }, { "label": "unstick mode", "value": "38%", "hint": "answer-key coverage on stuck-agent consultations", "sort": 0.37878787878787873 }, { "label": "challenge mode", "value": "64%", "hint": "stance-gated grounding on user challenges", "sort": 0.6388888888888888 }, { "label": "stance accuracy", "value": "8/12", "hint": "concede when the user is right, hold when the user is wrong", "sort": 0.6666666666666666 }, { "label": "file recall", "value": "85%", "hint": "share of the real fix's files named in upfront plans — diagnostic, not part of the score", "sort": 0.8537037037037036 }, { "label": "answered", "value": "32/32", "hint": "successful structured advisor calls", "sort": null }, { "label": "latency p50/p95", "value": "64.3s / 180.5s", "hint": "wall-clock per attempted advisor call; failures count as the 5-minute timeout", "sort": null } ] }, { "model": "moonshotai/kimi-k3@high", "qualityScore": 65, "latencyP50Ms": 79860, "latencyP95Ms": 300000, "totalDurationMs": 4117178, "costUsd": 0.320868, "costPerCaseUsd": 0.010027125, "tokens": 106956, "casesAttempted": 32, "metrics": [ { "label": "plan mode", "value": "86%", "hint": "answer-key coverage on upfront plan requests", "sort": 0.8611111111111112 }, { "label": "unstick mode", "value": "44%", "hint": "answer-key coverage on stuck-agent consultations", "sort": 0.4393939393939394 }, { "label": "challenge mode", "value": "64%", "hint": "stance-gated grounding on user challenges", "sort": 0.638888888888889 }, { "label": "stance accuracy", "value": "8/11", "hint": "concede when the user is right, hold when the user is wrong", "sort": 0.7272727272727273 }, { "label": "file recall", "value": "59%", "hint": "share of the real fix's files named in upfront plans — diagnostic, not part of the score", "sort": 0.5925925925925926 }, { "label": "answered", "value": "28/32", "hint": "successful structured advisor calls", "sort": null }, { "label": "latency p50/p95", "value": "79.9s / 300.0s", "hint": "wall-clock per attempted advisor call; failures count as the 5-minute timeout", "sort": null } ] }, { "model": "deepseek/deepseek-v4-pro@high", "qualityScore": 62, "latencyP50Ms": 48435, "latencyP95Ms": 95166, "totalDurationMs": 1720368, "costUsd": 0.0365922, "costPerCaseUsd": 0.00114350625, "tokens": 84120, "casesAttempted": 32, "metrics": [ { "label": "plan mode", "value": "72%", "hint": "answer-key coverage on upfront plan requests", "sort": 0.7222222222222222 }, { "label": "unstick mode", "value": "53%", "hint": "answer-key coverage on stuck-agent consultations", "sort": 0.5303030303030303 }, { "label": "challenge mode", "value": "61%", "hint": "stance-gated grounding on user challenges", "sort": 0.611111111111111 }, { "label": "stance accuracy", "value": "8/12", "hint": "concede when the user is right, hold when the user is wrong", "sort": 0.6666666666666666 }, { "label": "file recall", "value": "37%", "hint": "share of the real fix's files named in upfront plans — diagnostic, not part of the score", "sort": 0.3703703703703704 }, { "label": "answered", "value": "31/32", "hint": "successful structured advisor calls", "sort": null }, { "label": "latency p50/p95", "value": "48.4s / 95.2s", "hint": "wall-clock per attempted advisor call; failures count as the 5-minute timeout", "sort": null } ] }, { "model": "moonshotai/kimi-k2.7-code@high", "qualityScore": 61, "latencyP50Ms": 56956, "latencyP95Ms": 304950, "totalDurationMs": 3621837, "costUsd": 0.12064079999999999, "costPerCaseUsd": 0.0037700249999999998, "tokens": 172344, "casesAttempted": 32, "metrics": [ { "label": "plan mode", "value": "81%", "hint": "answer-key coverage on upfront plan requests", "sort": 0.8074074074074074 }, { "label": "unstick mode", "value": "48%", "hint": "answer-key coverage on stuck-agent consultations", "sort": 0.48484848484848475 }, { "label": "challenge mode", "value": "53%", "hint": "stance-gated grounding on user challenges", "sort": 0.5277777777777778 }, { "label": "stance accuracy", "value": "7/11", "hint": "concede when the user is right, hold when the user is wrong", "sort": 0.6363636363636364 }, { "label": "file recall", "value": "51%", "hint": "share of the real fix's files named in upfront plans — diagnostic, not part of the score", "sort": 0.5129629629629631 }, { "label": "answered", "value": "29/32", "hint": "successful structured advisor calls", "sort": null }, { "label": "latency p50/p95", "value": "57.0s / 305.0s", "hint": "wall-clock per attempted advisor call; failures count as the 5-minute timeout", "sort": null } ] }, { "model": "anthropic/claude-haiku-4.5@high", "qualityScore": 51, "latencyP50Ms": 49382, "latencyP95Ms": 300000, "totalDurationMs": 2943609, "costUsd": 0.132235, "costPerCaseUsd": 0.00413234375, "tokens": 132235, "casesAttempted": 32, "metrics": [ { "label": "plan mode", "value": "64%", "hint": "answer-key coverage on upfront plan requests", "sort": 0.6388888888888888 }, { "label": "unstick mode", "value": "36%", "hint": "answer-key coverage on stuck-agent consultations", "sort": 0.36363636363636365 }, { "label": "challenge mode", "value": "53%", "hint": "stance-gated grounding on user challenges", "sort": 0.5277777777777778 }, { "label": "stance accuracy", "value": "7/11", "hint": "concede when the user is right, hold when the user is wrong", "sort": 0.6363636363636364 }, { "label": "file recall", "value": "35%", "hint": "share of the real fix's files named in upfront plans — diagnostic, not part of the score", "sort": 0.35 }, { "label": "answered", "value": "27/32", "hint": "successful structured advisor calls", "sort": null }, { "label": "latency p50/p95", "value": "49.4s / 300.0s", "hint": "wall-clock per attempted advisor call; failures count as the 5-minute timeout", "sort": null } ] }, { "model": "minimax/minimax-m3@high", "qualityScore": 49, "latencyP50Ms": 34180, "latencyP95Ms": 300000, "totalDurationMs": 2261505, "costUsd": 0.0140082, "costPerCaseUsd": 0.00043775625, "tokens": 46694, "casesAttempted": 32, "metrics": [ { "label": "plan mode", "value": "58%", "hint": "answer-key coverage on upfront plan requests", "sort": 0.5833333333333334 }, { "label": "unstick mode", "value": "50%", "hint": "answer-key coverage on stuck-agent consultations", "sort": 0.5 }, { "label": "challenge mode", "value": "39%", "hint": "stance-gated grounding on user challenges", "sort": 0.38888888888888884 }, { "label": "stance accuracy", "value": "5/11", "hint": "concede when the user is right, hold when the user is wrong", "sort": 0.45454545454545453 }, { "label": "file recall", "value": "21%", "hint": "share of the real fix's files named in upfront plans — diagnostic, not part of the score", "sort": 0.2148148148148148 }, { "label": "answered", "value": "27/32", "hint": "successful structured advisor calls", "sort": null }, { "label": "latency p50/p95", "value": "34.2s / 300.0s", "hint": "wall-clock per attempted advisor call; failures count as the 5-minute timeout", "sort": null } ] }, { "model": "z-ai/glm-5.2@high", "qualityScore": 45, "latencyP50Ms": 37950, "latencyP95Ms": 217835, "totalDurationMs": 1842682, "costUsd": 0.0293158656, "costPerCaseUsd": 0.0009161208, "tokens": 72708, "casesAttempted": 32, "metrics": [ { "label": "plan mode", "value": "34%", "hint": "answer-key coverage on upfront plan requests", "sort": 0.34259259259259256 }, { "label": "unstick mode", "value": "58%", "hint": "answer-key coverage on stuck-agent consultations", "sort": 0.5757575757575757 }, { "label": "challenge mode", "value": "44%", "hint": "stance-gated grounding on user challenges", "sort": 0.4444444444444444 }, { "label": "stance accuracy", "value": "7/12", "hint": "concede when the user is right, hold when the user is wrong", "sort": 0.5833333333333334 }, { "label": "file recall", "value": "18%", "hint": "share of the real fix's files named in upfront plans — diagnostic, not part of the score", "sort": 0.17777777777777778 }, { "label": "answered", "value": "29/32", "hint": "successful structured advisor calls", "sort": null }, { "label": "latency p50/p95", "value": "38.0s / 217.8s", "hint": "wall-clock per attempted advisor call; failures count as the 5-minute timeout", "sort": null } ] }, { "model": "qwen/qwen3.8-max@high", "qualityScore": 39, "latencyP50Ms": 219490, "latencyP95Ms": 300000, "totalDurationMs": 6543038, "costUsd": 0.333378, "costPerCaseUsd": 0.0104180625, "tokens": 166689, "casesAttempted": 32, "metrics": [ { "label": "plan mode", "value": "61%", "hint": "answer-key coverage on upfront plan requests", "sort": 0.6111111111111112 }, { "label": "unstick mode", "value": "23%", "hint": "answer-key coverage on stuck-agent consultations", "sort": 0.22727272727272727 }, { "label": "challenge mode", "value": "33%", "hint": "stance-gated grounding on user challenges", "sort": 0.3333333333333333 }, { "label": "stance accuracy", "value": "4/9", "hint": "concede when the user is right, hold when the user is wrong", "sort": 0.4444444444444444 }, { "label": "file recall", "value": "45%", "hint": "share of the real fix's files named in upfront plans — diagnostic, not part of the score", "sort": 0.45185185185185184 }, { "label": "answered", "value": "23/32", "hint": "successful structured advisor calls", "sort": null }, { "label": "latency p50/p95", "value": "219.5s / 300.0s", "hint": "wall-clock per attempted advisor call; failures count as the 5-minute timeout", "sort": null } ] }, { "model": "deepseek/deepseek-v4-flash-0731@high", "qualityScore": 35, "latencyP50Ms": 86673, "latencyP95Ms": 300000, "totalDurationMs": 4767600, "costUsd": 0.00666747, "costPerCaseUsd": 0.0002083584375, "tokens": 74083, "casesAttempted": 32, "metrics": [ { "label": "plan mode", "value": "28%", "hint": "answer-key coverage on upfront plan requests", "sort": 0.2777777777777778 }, { "label": "unstick mode", "value": "38%", "hint": "answer-key coverage on stuck-agent consultations", "sort": 0.37878787878787873 }, { "label": "challenge mode", "value": "39%", "hint": "stance-gated grounding on user challenges", "sort": 0.3888888888888889 }, { "label": "stance accuracy", "value": "5/7", "hint": "concede when the user is right, hold when the user is wrong", "sort": 0.7142857142857143 }, { "label": "file recall", "value": "17%", "hint": "share of the real fix's files named in upfront plans — diagnostic, not part of the score", "sort": 0.16666666666666666 }, { "label": "answered", "value": "20/32", "hint": "successful structured advisor calls", "sort": null }, { "label": "latency p50/p95", "value": "86.7s / 300.0s", "hint": "wall-clock per attempted advisor call; failures count as the 5-minute timeout", "sort": null } ] }, { "model": "deepseek/deepseek-v4-pro-0813@high", "qualityScore": 34, "latencyP50Ms": 109158, "latencyP95Ms": 300000, "totalDurationMs": 4580865, "costUsd": 0.07583004, "costPerCaseUsd": 0.00236968875, "tokens": 114894, "casesAttempted": 32, "metrics": [ { "label": "plan mode", "value": "32%", "hint": "answer-key coverage on upfront plan requests", "sort": 0.32407407407407407 }, { "label": "unstick mode", "value": "27%", "hint": "answer-key coverage on stuck-agent consultations", "sort": 0.2727272727272727 }, { "label": "challenge mode", "value": "42%", "hint": "stance-gated grounding on user challenges", "sort": 0.4166666666666667 }, { "label": "stance accuracy", "value": "8/10", "hint": "concede when the user is right, hold when the user is wrong", "sort": 0.8 }, { "label": "file recall", "value": "16%", "hint": "share of the real fix's files named in upfront plans — diagnostic, not part of the score", "sort": 0.1611111111111111 }, { "label": "answered", "value": "20/32", "hint": "successful structured advisor calls", "sort": null }, { "label": "latency p50/p95", "value": "109.2s / 300.0s", "hint": "wall-clock per attempted advisor call; failures count as the 5-minute timeout", "sort": null } ] }, { "model": "thinkingmachines/inkling@high", "qualityScore": 23, "latencyP50Ms": 300000, "latencyP95Ms": 300000, "totalDurationMs": 6654461, "costUsd": 0.04948835, "costPerCaseUsd": 0.0015465109375, "tokens": 52093, "casesAttempted": 32, "metrics": [ { "label": "plan mode", "value": "31%", "hint": "answer-key coverage on upfront plan requests", "sort": 0.3055555555555556 }, { "label": "unstick mode", "value": "21%", "hint": "answer-key coverage on stuck-agent consultations", "sort": 0.2121212121212121 }, { "label": "challenge mode", "value": "17%", "hint": "stance-gated grounding on user challenges", "sort": 0.16666666666666666 }, { "label": "stance accuracy", "value": "2/5", "hint": "concede when the user is right, hold when the user is wrong", "sort": 0.4 }, { "label": "file recall", "value": "26%", "hint": "share of the real fix's files named in upfront plans — diagnostic, not part of the score", "sort": 0.2592592592592593 }, { "label": "answered", "value": "14/32", "hint": "successful structured advisor calls", "sort": null }, { "label": "latency p50/p95", "value": "300.0s / 300.0s", "hint": "wall-clock per attempted advisor call; failures count as the 5-minute timeout", "sort": null } ] } ] } ] }