{
  "benchmark": "LLMRouterBench (performance-cost, flagship pool)",
  "router": "dev evidence only (no-Jev ablation)",
  "n_eval": 5835,
  "n_dev": 5833,
  "seed": 20260917,
  "pool": [
    "claude-sonnet-4",
    "deepseek-v3-0324",
    "deepseek-v3.1-terminus",
    "deepseek-r1-0528",
    "gemini-2.5-flash",
    "gemini-2.5-pro",
    "gpt-5-chat",
    "gpt-5",
    "qwen3-235b-a22b-2507",
    "qwen3-235b-a22b-thinking-2507",
    "glm-4.6",
    "kimi-k2-0905",
    "intern-s1"
  ],
  "datasets": [
    "aime",
    "arenahard",
    "gpqa",
    "hle",
    "livecodebench",
    "livemathbench",
    "mmlupro",
    "simpleqa"
  ],
  "best_single": {
    "model": "gpt-5",
    "accuracy": 0.6029991431019709,
    "cost_per_1k": 31.55087210796915
  },
  "oracle": {
    "accuracy": 0.8262210796915167,
    "cost_per_query": 0.004829501106255356,
    "cost_per_1k": 4.829501106255356
  },
  "single_models": {
    "claude-sonnet-4": {
      "accuracy": 0.381662382176521,
      "cost_per_query": 0.007718803598971723,
      "cost_per_1k": 7.718803598971722
    },
    "deepseek-v3-0324": {
      "accuracy": 0.42510711225364184,
      "cost_per_query": 0.000596749263067695,
      "cost_per_1k": 0.5967492630676949
    },
    "deepseek-v3.1-terminus": {
      "accuracy": 0.44978577549271637,
      "cost_per_query": 0.0008278252733504712,
      "cost_per_1k": 0.8278252733504712
    },
    "deepseek-r1-0528": {
      "accuracy": 0.4875749785775493,
      "cost_per_query": 0.012752030034275922,
      "cost_per_1k": 12.752030034275922
    },
    "gemini-2.5-flash": {
      "accuracy": 0.44447300771208226,
      "cost_per_query": 0.008722070095972579,
      "cost_per_1k": 8.72207009597258
    },
    "gemini-2.5-pro": {
      "accuracy": 0.6023993144815767,
      "cost_per_query": 0.05534571266066838,
      "cost_per_1k": 55.34571266066838
    },
    "gpt-5-chat": {
      "accuracy": 0.4928877463581834,
      "cost_per_query": 0.004173646079691517,
      "cost_per_1k": 4.173646079691517
    },
    "gpt-5": {
      "accuracy": 0.6029991431019709,
      "cost_per_query": 0.03155087210796915,
      "cost_per_1k": 31.55087210796915
    },
    "qwen3-235b-a22b-2507": {
      "accuracy": 0.5522707797772065,
      "cost_per_query": 0.0007627481508140531,
      "cost_per_1k": 0.7627481508140531
    },
    "qwen3-235b-a22b-thinking-2507": {
      "accuracy": 0.537617823479006,
      "cost_per_query": 0.007077764260497,
      "cost_per_1k": 7.077764260497
    },
    "glm-4.6": {
      "accuracy": 0.44781491002570695,
      "cost_per_query": 0.012662073401885176,
      "cost_per_1k": 12.662073401885175
    },
    "kimi-k2-0905": {
      "accuracy": 0.4590402742073693,
      "cost_per_query": 0.001210231588688946,
      "cost_per_1k": 1.210231588688946
    },
    "intern-s1": {
      "accuracy": 0.3929734361610968,
      "cost_per_query": 0.002689900812339332,
      "cost_per_1k": 2.689900812339332
    }
  },
  "frontier": [
    {
      "cost_weight": 0.0,
      "accuracy": 0.62382176520994,
      "cost_per_1k": 30.45871833933162,
      "selection_distribution": {
        "claude-sonnet-4": 357,
        "deepseek-v3-0324": 34,
        "deepseek-v3.1-terminus": 340,
        "deepseek-r1-0528": 383,
        "gemini-2.5-flash": 81,
        "gemini-2.5-pro": 1754,
        "gpt-5-chat": 107,
        "gpt-5": 1520,
        "qwen3-235b-a22b-2507": 899,
        "qwen3-235b-a22b-thinking-2507": 202,
        "glm-4.6": 39,
        "kimi-k2-0905": 96,
        "intern-s1": 23
      }
    },
    {
      "cost_weight": 0.05,
      "accuracy": 0.6244215938303342,
      "cost_per_1k": 26.693428499571553,
      "selection_distribution": {
        "claude-sonnet-4": 179,
        "deepseek-v3-0324": 46,
        "deepseek-v3.1-terminus": 320,
        "deepseek-r1-0528": 250,
        "gemini-2.5-flash": 74,
        "gemini-2.5-pro": 1266,
        "gpt-5-chat": 132,
        "gpt-5": 1614,
        "qwen3-235b-a22b-2507": 1314,
        "qwen3-235b-a22b-thinking-2507": 342,
        "glm-4.6": 89,
        "kimi-k2-0905": 158,
        "intern-s1": 51
      }
    },
    {
      "cost_weight": 0.1,
      "accuracy": 0.6241645244215939,
      "cost_per_1k": 26.649430072836335,
      "selection_distribution": {
        "claude-sonnet-4": 179,
        "deepseek-v3-0324": 46,
        "deepseek-v3.1-terminus": 321,
        "deepseek-r1-0528": 250,
        "gemini-2.5-flash": 74,
        "gemini-2.5-pro": 1258,
        "gpt-5-chat": 133,
        "gpt-5": 1614,
        "qwen3-235b-a22b-2507": 1317,
        "qwen3-235b-a22b-thinking-2507": 343,
        "glm-4.6": 89,
        "kimi-k2-0905": 160,
        "intern-s1": 51
      }
    },
    {
      "cost_weight": 0.15,
      "accuracy": 0.6235646958011997,
      "cost_per_1k": 26.438142307626393,
      "selection_distribution": {
        "claude-sonnet-4": 179,
        "deepseek-v3-0324": 50,
        "deepseek-v3.1-terminus": 322,
        "deepseek-r1-0528": 253,
        "gemini-2.5-flash": 82,
        "gemini-2.5-pro": 1159,
        "gpt-5-chat": 136,
        "gpt-5": 1614,
        "qwen3-235b-a22b-2507": 1382,
        "qwen3-235b-a22b-thinking-2507": 354,
        "glm-4.6": 90,
        "kimi-k2-0905": 163,
        "intern-s1": 51
      }
    },
    {
      "cost_weight": 0.2,
      "accuracy": 0.6226221079691516,
      "cost_per_1k": 26.07214150899743,
      "selection_distribution": {
        "claude-sonnet-4": 179,
        "deepseek-v3-0324": 50,
        "deepseek-v3.1-terminus": 322,
        "deepseek-r1-0528": 260,
        "gemini-2.5-flash": 84,
        "gemini-2.5-pro": 1081,
        "gpt-5-chat": 142,
        "gpt-5": 1609,
        "qwen3-235b-a22b-2507": 1416,
        "qwen3-235b-a22b-thinking-2507": 386,
        "glm-4.6": 90,
        "kimi-k2-0905": 165,
        "intern-s1": 51
      }
    },
    {
      "cost_weight": 0.3,
      "accuracy": 0.6206512425021422,
      "cost_per_1k": 24.786372290488433,
      "selection_distribution": {
        "claude-sonnet-4": 182,
        "deepseek-v3-0324": 56,
        "deepseek-v3.1-terminus": 333,
        "deepseek-r1-0528": 277,
        "gemini-2.5-flash": 89,
        "gemini-2.5-pro": 896,
        "gpt-5-chat": 156,
        "gpt-5": 1575,
        "qwen3-235b-a22b-2507": 1523,
        "qwen3-235b-a22b-thinking-2507": 430,
        "glm-4.6": 92,
        "kimi-k2-0905": 171,
        "intern-s1": 55
      }
    },
    {
      "cost_weight": 0.4,
      "accuracy": 0.6169665809768637,
      "cost_per_1k": 23.552375167952015,
      "selection_distribution": {
        "claude-sonnet-4": 188,
        "deepseek-v3-0324": 65,
        "deepseek-v3.1-terminus": 340,
        "deepseek-r1-0528": 302,
        "gemini-2.5-flash": 94,
        "gemini-2.5-pro": 720,
        "gpt-5-chat": 177,
        "gpt-5": 1539,
        "qwen3-235b-a22b-2507": 1590,
        "qwen3-235b-a22b-thinking-2507": 483,
        "glm-4.6": 100,
        "kimi-k2-0905": 176,
        "intern-s1": 61
      }
    },
    {
      "cost_weight": 0.5,
      "accuracy": 0.614567266495287,
      "cost_per_1k": 22.553681842330764,
      "selection_distribution": {
        "claude-sonnet-4": 194,
        "deepseek-v3-0324": 67,
        "deepseek-v3.1-terminus": 360,
        "deepseek-r1-0528": 331,
        "gemini-2.5-flash": 96,
        "gemini-2.5-pro": 554,
        "gpt-5-chat": 200,
        "gpt-5": 1472,
        "qwen3-235b-a22b-2507": 1686,
        "qwen3-235b-a22b-thinking-2507": 516,
        "glm-4.6": 104,
        "kimi-k2-0905": 187,
        "intern-s1": 68
      }
    },
    {
      "cost_weight": 0.65,
      "accuracy": 0.6119108826049701,
      "cost_per_1k": 20.904456412596403,
      "selection_distribution": {
        "claude-sonnet-4": 201,
        "deepseek-v3-0324": 73,
        "deepseek-v3.1-terminus": 385,
        "deepseek-r1-0528": 383,
        "gemini-2.5-flash": 102,
        "gemini-2.5-pro": 337,
        "gpt-5-chat": 226,
        "gpt-5": 1319,
        "qwen3-235b-a22b-2507": 1830,
        "qwen3-235b-a22b-thinking-2507": 584,
        "glm-4.6": 110,
        "kimi-k2-0905": 211,
        "intern-s1": 74
      }
    },
    {
      "cost_weight": 0.8,
      "accuracy": 0.606426735218509,
      "cost_per_1k": 19.470890559554416,
      "selection_distribution": {
        "claude-sonnet-4": 212,
        "deepseek-v3-0324": 78,
        "deepseek-v3.1-terminus": 407,
        "deepseek-r1-0528": 411,
        "gemini-2.5-flash": 101,
        "gemini-2.5-pro": 206,
        "gpt-5-chat": 265,
        "gpt-5": 1202,
        "qwen3-235b-a22b-2507": 1924,
        "qwen3-235b-a22b-thinking-2507": 609,
        "glm-4.6": 113,
        "kimi-k2-0905": 225,
        "intern-s1": 82
      }
    },
    {
      "cost_weight": 1.0,
      "accuracy": 0.6040274207369323,
      "cost_per_1k": 17.93375446101114,
      "selection_distribution": {
        "claude-sonnet-4": 217,
        "deepseek-v3-0324": 106,
        "deepseek-v3.1-terminus": 433,
        "deepseek-r1-0528": 458,
        "gemini-2.5-flash": 84,
        "gemini-2.5-pro": 91,
        "gpt-5-chat": 295,
        "gpt-5": 1028,
        "qwen3-235b-a22b-2507": 2028,
        "qwen3-235b-a22b-thinking-2507": 631,
        "glm-4.6": 129,
        "kimi-k2-0905": 246,
        "intern-s1": 89
      }
    },
    {
      "cost_weight": 1.25,
      "accuracy": 0.6007712082262211,
      "cost_per_1k": 15.842605658097687,
      "selection_distribution": {
        "claude-sonnet-4": 220,
        "deepseek-v3-0324": 122,
        "deepseek-v3.1-terminus": 484,
        "deepseek-r1-0528": 431,
        "gemini-2.5-flash": 85,
        "gemini-2.5-pro": 26,
        "gpt-5-chat": 318,
        "gpt-5": 834,
        "qwen3-235b-a22b-2507": 2164,
        "qwen3-235b-a22b-thinking-2507": 639,
        "glm-4.6": 139,
        "kimi-k2-0905": 282,
        "intern-s1": 91
      }
    },
    {
      "cost_weight": 1.5,
      "accuracy": 0.5923736075407027,
      "cost_per_1k": 14.025975029134534,
      "selection_distribution": {
        "claude-sonnet-4": 222,
        "deepseek-v3-0324": 136,
        "deepseek-v3.1-terminus": 518,
        "deepseek-r1-0528": 444,
        "gemini-2.5-flash": 85,
        "gemini-2.5-pro": 3,
        "gpt-5-chat": 333,
        "gpt-5": 645,
        "qwen3-235b-a22b-2507": 2276,
        "qwen3-235b-a22b-thinking-2507": 620,
        "glm-4.6": 143,
        "kimi-k2-0905": 312,
        "intern-s1": 98
      }
    },
    {
      "cost_weight": 2.0,
      "accuracy": 0.5783204798628964,
      "cost_per_1k": 9.57967500856898,
      "selection_distribution": {
        "claude-sonnet-4": 215,
        "deepseek-v3-0324": 185,
        "deepseek-v3.1-terminus": 669,
        "deepseek-r1-0528": 376,
        "gemini-2.5-flash": 72,
        "gpt-5-chat": 328,
        "gpt-5": 296,
        "qwen3-235b-a22b-2507": 2478,
        "qwen3-235b-a22b-thinking-2507": 604,
        "glm-4.6": 128,
        "kimi-k2-0905": 360,
        "intern-s1": 124
      }
    }
  ]
}