{
  "benchmark": "LLMRouterBench (performance-cost, flagship pool)",
  "router": "Jev difficulty + dev evidence",
  "n_eval": 5835,
  "n_dev": 5833,
  "seed": 20260917,
  "pool": [
    "claude-sonnet-4",
    "deepseek-v3-0324",
    "deepseek-v3.1-terminus",
    "deepseek-r1-0528",
    "gemini-2.5-flash",
    "gemini-2.5-pro",
    "gpt-5-chat",
    "gpt-5",
    "qwen3-235b-a22b-2507",
    "qwen3-235b-a22b-thinking-2507",
    "glm-4.6",
    "kimi-k2-0905",
    "intern-s1"
  ],
  "datasets": [
    "aime",
    "arenahard",
    "gpqa",
    "hle",
    "livecodebench",
    "livemathbench",
    "mmlupro",
    "simpleqa"
  ],
  "best_single": {
    "model": "gpt-5",
    "accuracy": 0.6029991431019709,
    "cost_per_1k": 31.55087210796915
  },
  "oracle": {
    "accuracy": 0.8262210796915167,
    "cost_per_query": 0.004829501106255356,
    "cost_per_1k": 4.829501106255356
  },
  "single_models": {
    "claude-sonnet-4": {
      "accuracy": 0.381662382176521,
      "cost_per_query": 0.007718803598971723,
      "cost_per_1k": 7.718803598971722
    },
    "deepseek-v3-0324": {
      "accuracy": 0.42510711225364184,
      "cost_per_query": 0.000596749263067695,
      "cost_per_1k": 0.5967492630676949
    },
    "deepseek-v3.1-terminus": {
      "accuracy": 0.44978577549271637,
      "cost_per_query": 0.0008278252733504712,
      "cost_per_1k": 0.8278252733504712
    },
    "deepseek-r1-0528": {
      "accuracy": 0.4875749785775493,
      "cost_per_query": 0.012752030034275922,
      "cost_per_1k": 12.752030034275922
    },
    "gemini-2.5-flash": {
      "accuracy": 0.44447300771208226,
      "cost_per_query": 0.008722070095972579,
      "cost_per_1k": 8.72207009597258
    },
    "gemini-2.5-pro": {
      "accuracy": 0.6023993144815767,
      "cost_per_query": 0.05534571266066838,
      "cost_per_1k": 55.34571266066838
    },
    "gpt-5-chat": {
      "accuracy": 0.4928877463581834,
      "cost_per_query": 0.004173646079691517,
      "cost_per_1k": 4.173646079691517
    },
    "gpt-5": {
      "accuracy": 0.6029991431019709,
      "cost_per_query": 0.03155087210796915,
      "cost_per_1k": 31.55087210796915
    },
    "qwen3-235b-a22b-2507": {
      "accuracy": 0.5522707797772065,
      "cost_per_query": 0.0007627481508140531,
      "cost_per_1k": 0.7627481508140531
    },
    "qwen3-235b-a22b-thinking-2507": {
      "accuracy": 0.537617823479006,
      "cost_per_query": 0.007077764260497,
      "cost_per_1k": 7.077764260497
    },
    "glm-4.6": {
      "accuracy": 0.44781491002570695,
      "cost_per_query": 0.012662073401885176,
      "cost_per_1k": 12.662073401885175
    },
    "kimi-k2-0905": {
      "accuracy": 0.4590402742073693,
      "cost_per_query": 0.001210231588688946,
      "cost_per_1k": 1.210231588688946
    },
    "intern-s1": {
      "accuracy": 0.3929734361610968,
      "cost_per_query": 0.002689900812339332,
      "cost_per_1k": 2.689900812339332
    }
  },
  "frontier": [
    {
      "cost_weight": 0.0,
      "accuracy": 0.62382176520994,
      "cost_per_1k": 30.45871833933162,
      "selection_distribution": {
        "claude-sonnet-4": 357,
        "deepseek-v3-0324": 34,
        "deepseek-v3.1-terminus": 340,
        "deepseek-r1-0528": 383,
        "gemini-2.5-flash": 81,
        "gemini-2.5-pro": 1754,
        "gpt-5-chat": 107,
        "gpt-5": 1520,
        "qwen3-235b-a22b-2507": 899,
        "qwen3-235b-a22b-thinking-2507": 202,
        "glm-4.6": 39,
        "kimi-k2-0905": 96,
        "intern-s1": 23
      }
    },
    {
      "cost_weight": 0.05,
      "accuracy": 0.6242502142245073,
      "cost_per_1k": 26.734926550985435,
      "selection_distribution": {
        "claude-sonnet-4": 179,
        "deepseek-v3-0324": 46,
        "deepseek-v3.1-terminus": 320,
        "deepseek-r1-0528": 250,
        "gemini-2.5-flash": 74,
        "gemini-2.5-pro": 1270,
        "gpt-5-chat": 132,
        "gpt-5": 1609,
        "qwen3-235b-a22b-2507": 1315,
        "qwen3-235b-a22b-thinking-2507": 342,
        "glm-4.6": 89,
        "kimi-k2-0905": 158,
        "intern-s1": 51
      }
    },
    {
      "cost_weight": 0.1,
      "accuracy": 0.6227077977720651,
      "cost_per_1k": 26.525272746358187,
      "selection_distribution": {
        "claude-sonnet-4": 179,
        "deepseek-v3-0324": 50,
        "deepseek-v3.1-terminus": 320,
        "deepseek-r1-0528": 250,
        "gemini-2.5-flash": 84,
        "gemini-2.5-pro": 1135,
        "gpt-5-chat": 139,
        "gpt-5": 1607,
        "qwen3-235b-a22b-2507": 1399,
        "qwen3-235b-a22b-thinking-2507": 373,
        "glm-4.6": 89,
        "kimi-k2-0905": 159,
        "intern-s1": 51
      }
    },
    {
      "cost_weight": 0.15,
      "accuracy": 0.6223650385604114,
      "cost_per_1k": 26.319403080548412,
      "selection_distribution": {
        "claude-sonnet-4": 179,
        "deepseek-v3-0324": 55,
        "deepseek-v3.1-terminus": 321,
        "deepseek-r1-0528": 250,
        "gemini-2.5-flash": 87,
        "gemini-2.5-pro": 1032,
        "gpt-5-chat": 152,
        "gpt-5": 1588,
        "qwen3-235b-a22b-2507": 1478,
        "qwen3-235b-a22b-thinking-2507": 394,
        "glm-4.6": 89,
        "kimi-k2-0905": 159,
        "intern-s1": 51
      }
    },
    {
      "cost_weight": 0.2,
      "accuracy": 0.6213367609254499,
      "cost_per_1k": 26.118908487574977,
      "selection_distribution": {
        "claude-sonnet-4": 180,
        "deepseek-v3-0324": 61,
        "deepseek-v3.1-terminus": 321,
        "deepseek-r1-0528": 250,
        "gemini-2.5-flash": 90,
        "gemini-2.5-pro": 942,
        "gpt-5-chat": 163,
        "gpt-5": 1575,
        "qwen3-235b-a22b-2507": 1534,
        "qwen3-235b-a22b-thinking-2507": 416,
        "glm-4.6": 90,
        "kimi-k2-0905": 162,
        "intern-s1": 51
      }
    },
    {
      "cost_weight": 0.3,
      "accuracy": 0.6155098543273351,
      "cost_per_1k": 25.53948564353042,
      "selection_distribution": {
        "claude-sonnet-4": 182,
        "deepseek-v3-0324": 65,
        "deepseek-v3.1-terminus": 332,
        "deepseek-r1-0528": 258,
        "gemini-2.5-flash": 99,
        "gemini-2.5-pro": 700,
        "gpt-5-chat": 192,
        "gpt-5": 1536,
        "qwen3-235b-a22b-2507": 1693,
        "qwen3-235b-a22b-thinking-2507": 465,
        "glm-4.6": 91,
        "kimi-k2-0905": 170,
        "intern-s1": 52
      }
    },
    {
      "cost_weight": 0.4,
      "accuracy": 0.614738646101114,
      "cost_per_1k": 24.924270637103685,
      "selection_distribution": {
        "claude-sonnet-4": 189,
        "deepseek-v3-0324": 72,
        "deepseek-v3.1-terminus": 350,
        "deepseek-r1-0528": 266,
        "gemini-2.5-flash": 97,
        "gemini-2.5-pro": 543,
        "gpt-5-chat": 227,
        "gpt-5": 1469,
        "qwen3-235b-a22b-2507": 1806,
        "qwen3-235b-a22b-thinking-2507": 490,
        "glm-4.6": 91,
        "kimi-k2-0905": 179,
        "intern-s1": 56
      }
    },
    {
      "cost_weight": 0.5,
      "accuracy": 0.6141388174807197,
      "cost_per_1k": 24.48415718251928,
      "selection_distribution": {
        "claude-sonnet-4": 193,
        "deepseek-v3-0324": 94,
        "deepseek-v3.1-terminus": 360,
        "deepseek-r1-0528": 273,
        "gemini-2.5-flash": 78,
        "gemini-2.5-pro": 429,
        "gpt-5-chat": 255,
        "gpt-5": 1405,
        "qwen3-235b-a22b-2507": 1895,
        "qwen3-235b-a22b-thinking-2507": 515,
        "glm-4.6": 93,
        "kimi-k2-0905": 186,
        "intern-s1": 59
      }
    },
    {
      "cost_weight": 0.65,
      "accuracy": 0.6121679520137103,
      "cost_per_1k": 23.81327231362468,
      "selection_distribution": {
        "claude-sonnet-4": 201,
        "deepseek-v3-0324": 100,
        "deepseek-v3.1-terminus": 376,
        "deepseek-r1-0528": 281,
        "gemini-2.5-flash": 80,
        "gemini-2.5-pro": 348,
        "gpt-5-chat": 284,
        "gpt-5": 1311,
        "qwen3-235b-a22b-2507": 2005,
        "qwen3-235b-a22b-thinking-2507": 495,
        "glm-4.6": 95,
        "kimi-k2-0905": 195,
        "intern-s1": 64
      }
    },
    {
      "cost_weight": 0.8,
      "accuracy": 0.6087403598971722,
      "cost_per_1k": 23.4044523916024,
      "selection_distribution": {
        "claude-sonnet-4": 204,
        "deepseek-v3-0324": 107,
        "deepseek-v3.1-terminus": 389,
        "deepseek-r1-0528": 284,
        "gemini-2.5-flash": 78,
        "gemini-2.5-pro": 316,
        "gpt-5-chat": 302,
        "gpt-5": 1240,
        "qwen3-235b-a22b-2507": 2083,
        "qwen3-235b-a22b-thinking-2507": 465,
        "glm-4.6": 96,
        "kimi-k2-0905": 202,
        "intern-s1": 69
      }
    },
    {
      "cost_weight": 1.0,
      "accuracy": 0.6080548414738646,
      "cost_per_1k": 22.93538523993145,
      "selection_distribution": {
        "claude-sonnet-4": 206,
        "deepseek-v3-0324": 124,
        "deepseek-v3.1-terminus": 412,
        "deepseek-r1-0528": 292,
        "gemini-2.5-flash": 67,
        "gemini-2.5-pro": 278,
        "gpt-5-chat": 298,
        "gpt-5": 1179,
        "qwen3-235b-a22b-2507": 2155,
        "qwen3-235b-a22b-thinking-2507": 439,
        "glm-4.6": 98,
        "kimi-k2-0905": 211,
        "intern-s1": 76
      }
    },
    {
      "cost_weight": 1.25,
      "accuracy": 0.6061696658097686,
      "cost_per_1k": 22.430158375321337,
      "selection_distribution": {
        "claude-sonnet-4": 204,
        "deepseek-v3-0324": 139,
        "deepseek-v3.1-terminus": 441,
        "deepseek-r1-0528": 282,
        "gemini-2.5-flash": 63,
        "gemini-2.5-pro": 243,
        "gpt-5-chat": 281,
        "gpt-5": 1120,
        "qwen3-235b-a22b-2507": 2238,
        "qwen3-235b-a22b-thinking-2507": 424,
        "glm-4.6": 93,
        "kimi-k2-0905": 225,
        "intern-s1": 82
      }
    },
    {
      "cost_weight": 1.5,
      "accuracy": 0.6048843187660669,
      "cost_per_1k": 22.145493143958866,
      "selection_distribution": {
        "claude-sonnet-4": 200,
        "deepseek-v3-0324": 147,
        "deepseek-v3.1-terminus": 472,
        "deepseek-r1-0528": 265,
        "gemini-2.5-flash": 58,
        "gemini-2.5-pro": 229,
        "gpt-5-chat": 271,
        "gpt-5": 1074,
        "qwen3-235b-a22b-2507": 2329,
        "qwen3-235b-a22b-thinking-2507": 378,
        "glm-4.6": 88,
        "kimi-k2-0905": 239,
        "intern-s1": 85
      }
    },
    {
      "cost_weight": 2.0,
      "accuracy": 0.6032562125107113,
      "cost_per_1k": 21.09175394815767,
      "selection_distribution": {
        "claude-sonnet-4": 181,
        "deepseek-v3-0324": 156,
        "deepseek-v3.1-terminus": 539,
        "deepseek-r1-0528": 234,
        "gemini-2.5-flash": 43,
        "gemini-2.5-pro": 188,
        "gpt-5-chat": 256,
        "gpt-5": 1017,
        "qwen3-235b-a22b-2507": 2446,
        "qwen3-235b-a22b-thinking-2507": 341,
        "glm-4.6": 75,
        "kimi-k2-0905": 270,
        "intern-s1": 89
      }
    }
  ]
}