{
  "run_id": "2026-05-04T081941Z",
  "generated_at": "2026-05-04T08:19:41.006432+00:00",
  "models_tested": [
    "grok-3",
    "grok-3-mini",
    "grok-4-0709",
    "grok-4-1-fast-non-reasoning",
    "grok-4-1-fast-reasoning",
    "grok-4-fast-non-reasoning",
    "grok-4-fast-reasoning",
    "grok-4.20-0309-non-reasoning",
    "grok-4.20-0309-reasoning",
    "grok-4.3",
    "grok-code-fast-1"
  ],
  "models_skipped": [
    "grok-4.20-multi-agent-0309"
  ],
  "pricing": {
    "grok-3": {
      "model": "grok-3",
      "input_per_1m_usd": 3.0,
      "output_per_1m_usd": 15.0,
      "cached_input_per_1m_usd": 0.75
    },
    "grok-3-mini": {
      "model": "grok-3-mini",
      "input_per_1m_usd": 0.3,
      "output_per_1m_usd": 0.5,
      "cached_input_per_1m_usd": 0.075
    },
    "grok-4-0709": {
      "model": "grok-4-0709",
      "input_per_1m_usd": 3.0,
      "output_per_1m_usd": 15.0,
      "cached_input_per_1m_usd": 0.75
    },
    "grok-4-1-fast-non-reasoning": {
      "model": "grok-4-1-fast-non-reasoning",
      "input_per_1m_usd": 0.2,
      "output_per_1m_usd": 0.5,
      "cached_input_per_1m_usd": 0.05
    },
    "grok-4-1-fast-reasoning": {
      "model": "grok-4-1-fast-reasoning",
      "input_per_1m_usd": 0.2,
      "output_per_1m_usd": 0.5,
      "cached_input_per_1m_usd": 0.05
    },
    "grok-4-fast-non-reasoning": {
      "model": "grok-4-fast-non-reasoning",
      "input_per_1m_usd": 0.2,
      "output_per_1m_usd": 0.5,
      "cached_input_per_1m_usd": 0.05
    },
    "grok-4-fast-reasoning": {
      "model": "grok-4-fast-reasoning",
      "input_per_1m_usd": 0.2,
      "output_per_1m_usd": 0.5,
      "cached_input_per_1m_usd": 0.05
    },
    "grok-4.20-0309-non-reasoning": {
      "model": "grok-4.20-0309-non-reasoning",
      "input_per_1m_usd": 1.25,
      "output_per_1m_usd": 2.5,
      "cached_input_per_1m_usd": 0.2
    },
    "grok-4.20-0309-reasoning": {
      "model": "grok-4.20-0309-reasoning",
      "input_per_1m_usd": 1.25,
      "output_per_1m_usd": 2.5,
      "cached_input_per_1m_usd": 0.2
    },
    "grok-4.20-multi-agent-0309": {
      "model": "grok-4.20-multi-agent-0309",
      "input_per_1m_usd": 1.25,
      "output_per_1m_usd": 2.5,
      "cached_input_per_1m_usd": 0.2
    },
    "grok-4.3": {
      "model": "grok-4.3",
      "input_per_1m_usd": 1.25,
      "output_per_1m_usd": 2.5,
      "cached_input_per_1m_usd": 0.2
    },
    "grok-code-fast-1": {
      "model": "grok-code-fast-1",
      "input_per_1m_usd": 0.2,
      "output_per_1m_usd": 1.5,
      "cached_input_per_1m_usd": 0.02
    }
  },
  "records": [
    {
      "model": "grok-3",
      "task_id": "math_multiplication",
      "ok": false,
      "grader_msg": "missing_expected (got content: '1564642781')",
      "latency_ms": 428,
      "status": 200,
      "input_tokens": 40,
      "output_tokens": 5,
      "cached_tokens": 5,
      "reasoning_tokens": 0,
      "cost_usd": 0.000184,
      "content_preview": "1564642781",
      "tool_calls_count": 0,
      "error": null
    },
    {
      "model": "grok-3",
      "task_id": "tool_use_weather",
      "ok": true,
      "grader_msg": "called get_weather(city='paris', unit='celsius')",
      "latency_ms": 1069,
      "status": 200,
      "input_tokens": 299,
      "output_tokens": 33,
      "cached_tokens": 5,
      "reasoning_tokens": 0,
      "cost_usd": 0.001381,
      "content_preview": "",
      "tool_calls_count": 1,
      "error": null
    },
    {
      "model": "grok-3",
      "task_id": "code_is_prime",
      "ok": true,
      "grader_msg": "8/8 cases passed",
      "latency_ms": 1205,
      "status": 200,
      "input_tokens": 50,
      "output_tokens": 57,
      "cached_tokens": 3,
      "reasoning_tokens": 0,
      "cost_usd": 0.000998,
      "content_preview": "def is_prime(n: int) -> bool:\n    if n <= 1:\n        return False\n    for i in range(2, int(n ** 0.5) + 1):\n        if n % i == 0:\n            return False\n    return True",
      "tool_calls_count": 0,
      "error": null
    },
    {
      "model": "grok-3-mini",
      "task_id": "math_multiplication",
      "ok": true,
      "grader_msg": "exact",
      "latency_ms": 19584,
      "status": 200,
      "input_tokens": 41,
      "output_tokens": 4,
      "cached_tokens": 6,
      "reasoning_tokens": 1182,
      "cost_usd": 1.3e-05,
      "content_preview": "1564420181",
      "tool_calls_count": 0,
      "error": null
    },
    {
      "model": "grok-3-mini",
      "task_id": "tool_use_weather",
      "ok": true,
      "grader_msg": "called get_weather(city='paris', unit='celsius')",
      "latency_ms": 5127,
      "status": 200,
      "input_tokens": 322,
      "output_tokens": 33,
      "cached_tokens": 5,
      "reasoning_tokens": 277,
      "cost_usd": 0.000112,
      "content_preview": "",
      "tool_calls_count": 1,
      "error": null
    },
    {
      "model": "grok-3-mini",
      "task_id": "code_is_prime",
      "ok": true,
      "grader_msg": "8/8 cases passed",
      "latency_ms": 7719,
      "status": 200,
      "input_tokens": 51,
      "output_tokens": 56,
      "cached_tokens": 4,
      "reasoning_tokens": 589,
      "cost_usd": 4.2e-05,
      "content_preview": "def is_prime(n: int) -> bool:\n    if n <= 1:\n        return False\n    for i in range(2, int(n**0.5) + 1):\n        if n % i == 0:\n            return False\n    return True",
      "tool_calls_count": 0,
      "error": null
    },
    {
      "model": "grok-4-0709",
      "task_id": "math_multiplication",
      "ok": true,
      "grader_msg": "exact",
      "latency_ms": 27650,
      "status": 200,
      "input_tokens": 715,
      "output_tokens": 4,
      "cached_tokens": 679,
      "reasoning_tokens": 1091,
      "cost_usd": 0.000677,
      "content_preview": "1564420181",
      "tool_calls_count": 0,
      "error": null
    },
    {
      "model": "grok-4-0709",
      "task_id": "tool_use_weather",
      "ok": true,
      "grader_msg": "called get_weather(city='paris', unit='celsius')",
      "latency_ms": 5538,
      "status": 200,
      "input_tokens": 924,
      "output_tokens": 36,
      "cached_tokens": 679,
      "reasoning_tokens": 116,
      "cost_usd": 0.001784,
      "content_preview": "",
      "tool_calls_count": 1,
      "error": null
    },
    {
      "model": "grok-4-0709",
      "task_id": "code_is_prime",
      "ok": true,
      "grader_msg": "8/8 cases passed",
      "latency_ms": 20680,
      "status": 200,
      "input_tokens": 725,
      "output_tokens": 87,
      "cached_tokens": 677,
      "reasoning_tokens": 489,
      "cost_usd": 0.001957,
      "content_preview": "```python\ndef is_prime(n: int) -> bool:\n    if n <= 1:\n        return False\n    if n == 2:\n        return True\n    if n % 2 == 0:\n        return False\n    i = 3\n    while i * i <= n:\n        if n % i ",
      "tool_calls_count": 0,
      "error": null
    },
    {
      "model": "grok-4-1-fast-non-reasoning",
      "task_id": "math_multiplication",
      "ok": false,
      "grader_msg": "missing_expected (got content: '1564701381')",
      "latency_ms": 480,
      "status": 200,
      "input_tokens": 198,
      "output_tokens": 4,
      "cached_tokens": 161,
      "reasoning_tokens": 0,
      "cost_usd": 1.7e-05,
      "content_preview": "1564701381",
      "tool_calls_count": 0,
      "error": null
    },
    {
      "model": "grok-4-1-fast-non-reasoning",
      "task_id": "tool_use_weather",
      "ok": true,
      "grader_msg": "called get_weather(city='paris', unit='celsius')",
      "latency_ms": 2002,
      "status": 200,
      "input_tokens": 407,
      "output_tokens": 36,
      "cached_tokens": 160,
      "reasoning_tokens": 0,
      "cost_usd": 7.5e-05,
      "content_preview": "",
      "tool_calls_count": 1,
      "error": null
    },
    {
      "model": "grok-4-1-fast-non-reasoning",
      "task_id": "code_is_prime",
      "ok": true,
      "grader_msg": "8/8 cases passed",
      "latency_ms": 978,
      "status": 200,
      "input_tokens": 208,
      "output_tokens": 107,
      "cached_tokens": 158,
      "reasoning_tokens": 0,
      "cost_usd": 7.1e-05,
      "content_preview": "```python\ndef is_prime(n: int) -> bool:\n    if n <= 1:\n        return False\n    if n <= 3:\n        return True\n    if n % 2 == 0 or n % 3 == 0:\n        return False\n    i = 5\n    while i * i <= n:\n   ",
      "tool_calls_count": 0,
      "error": null
    },
    {
      "model": "grok-4-1-fast-reasoning",
      "task_id": "math_multiplication",
      "ok": true,
      "grader_msg": "exact",
      "latency_ms": 23127,
      "status": 200,
      "input_tokens": 186,
      "output_tokens": 4,
      "cached_tokens": 150,
      "reasoning_tokens": 2477,
      "cost_usd": 1.7e-05,
      "content_preview": "1564420181",
      "tool_calls_count": 0,
      "error": null
    },
    {
      "model": "grok-4-1-fast-reasoning",
      "task_id": "tool_use_weather",
      "ok": true,
      "grader_msg": "called get_weather(city='paris', unit='celsius')",
      "latency_ms": 2674,
      "status": 200,
      "input_tokens": 395,
      "output_tokens": 36,
      "cached_tokens": 150,
      "reasoning_tokens": 102,
      "cost_usd": 7.4e-05,
      "content_preview": "",
      "tool_calls_count": 1,
      "error": null
    },
    {
      "model": "grok-4-1-fast-reasoning",
      "task_id": "code_is_prime",
      "ok": true,
      "grader_msg": "8/8 cases passed",
      "latency_ms": 6627,
      "status": 200,
      "input_tokens": 196,
      "output_tokens": 103,
      "cached_tokens": 149,
      "reasoning_tokens": 633,
      "cost_usd": 6.8e-05,
      "content_preview": "```python\ndef is_prime(n: int) -> bool:\n    if n <= 1: return False\n    if n <= 3: return True\n    if n % 2 == 0 or n % 3 == 0: return False\n    i = 5\n    while i * i <= n:\n        if n % i == 0 or n ",
      "tool_calls_count": 0,
      "error": null
    },
    {
      "model": "grok-4-fast-non-reasoning",
      "task_id": "math_multiplication",
      "ok": false,
      "grader_msg": "missing_expected (got content: '1564998781')",
      "latency_ms": 447,
      "status": 200,
      "input_tokens": 198,
      "output_tokens": 4,
      "cached_tokens": 160,
      "reasoning_tokens": 0,
      "cost_usd": 1.8e-05,
      "content_preview": "1564998781",
      "tool_calls_count": 0,
      "error": null
    },
    {
      "model": "grok-4-fast-non-reasoning",
      "task_id": "tool_use_weather",
      "ok": true,
      "grader_msg": "called get_weather(city='paris', unit='celsius')",
      "latency_ms": 1764,
      "status": 200,
      "input_tokens": 407,
      "output_tokens": 36,
      "cached_tokens": 160,
      "reasoning_tokens": 0,
      "cost_usd": 7.5e-05,
      "content_preview": "",
      "tool_calls_count": 1,
      "error": null
    },
    {
      "model": "grok-4-fast-non-reasoning",
      "task_id": "code_is_prime",
      "ok": true,
      "grader_msg": "8/8 cases passed",
      "latency_ms": 1095,
      "status": 200,
      "input_tokens": 208,
      "output_tokens": 107,
      "cached_tokens": 158,
      "reasoning_tokens": 0,
      "cost_usd": 7.1e-05,
      "content_preview": "```python\ndef is_prime(n: int) -> bool:\n    if n <= 1:\n        return False\n    if n <= 3:\n        return True\n    if n % 2 == 0 or n % 3 == 0:\n        return False\n    i = 5\n    while i * i <= n:\n   ",
      "tool_calls_count": 0,
      "error": null
    },
    {
      "model": "grok-4-fast-reasoning",
      "task_id": "math_multiplication",
      "ok": true,
      "grader_msg": "exact",
      "latency_ms": 13788,
      "status": 200,
      "input_tokens": 186,
      "output_tokens": 4,
      "cached_tokens": 150,
      "reasoning_tokens": 1228,
      "cost_usd": 1.7e-05,
      "content_preview": "1564420181",
      "tool_calls_count": 0,
      "error": null
    },
    {
      "model": "grok-4-fast-reasoning",
      "task_id": "tool_use_weather",
      "ok": true,
      "grader_msg": "called get_weather(city='paris', unit='celsius')",
      "latency_ms": 2044,
      "status": 200,
      "input_tokens": 395,
      "output_tokens": 36,
      "cached_tokens": 150,
      "reasoning_tokens": 109,
      "cost_usd": 7.4e-05,
      "content_preview": "",
      "tool_calls_count": 1,
      "error": null
    },
    {
      "model": "grok-4-fast-reasoning",
      "task_id": "code_is_prime",
      "ok": true,
      "grader_msg": "8/8 cases passed",
      "latency_ms": 3244,
      "status": 200,
      "input_tokens": 196,
      "output_tokens": 87,
      "cached_tokens": 148,
      "reasoning_tokens": 250,
      "cost_usd": 6.1e-05,
      "content_preview": "```python\ndef is_prime(n: int) -> bool:\n    if n <= 1:\n        return False\n    if n == 2:\n        return True\n    if n % 2 == 0:\n        return False\n    i = 3\n    while i * i <= n:\n        if n % i ",
      "tool_calls_count": 0,
      "error": null
    },
    {
      "model": "grok-4.20-0309-non-reasoning",
      "task_id": "math_multiplication",
      "ok": false,
      "grader_msg": "missing_expected (got content: '1564493981')",
      "latency_ms": 481,
      "status": 200,
      "input_tokens": 155,
      "output_tokens": 4,
      "cached_tokens": 64,
      "reasoning_tokens": 0,
      "cost_usd": 0.000137,
      "content_preview": "1564493981",
      "tool_calls_count": 0,
      "error": null
    },
    {
      "model": "grok-4.20-0309-non-reasoning",
      "task_id": "tool_use_weather",
      "ok": true,
      "grader_msg": "called get_weather(city='paris', unit='celsius')",
      "latency_ms": 1361,
      "status": 200,
      "input_tokens": 264,
      "output_tokens": 18,
      "cached_tokens": 64,
      "reasoning_tokens": 0,
      "cost_usd": 0.000308,
      "content_preview": "",
      "tool_calls_count": 1,
      "error": null
    },
    {
      "model": "grok-4.20-0309-non-reasoning",
      "task_id": "code_is_prime",
      "ok": true,
      "grader_msg": "8/8 cases passed",
      "latency_ms": 1203,
      "status": 200,
      "input_tokens": 165,
      "output_tokens": 107,
      "cached_tokens": 64,
      "reasoning_tokens": 0,
      "cost_usd": 0.000407,
      "content_preview": "```python\ndef is_prime(n: int) -> bool:\n    if n <= 1:\n        return False\n    if n <= 3:\n        return True\n    if n % 2 == 0 or n % 3 == 0:\n        return False\n    i = 5\n    while i * i <= n:\n   ",
      "tool_calls_count": 0,
      "error": null
    },
    {
      "model": "grok-4.20-0309-reasoning",
      "task_id": "math_multiplication",
      "ok": true,
      "grader_msg": "exact",
      "latency_ms": 22737,
      "status": 200,
      "input_tokens": 157,
      "output_tokens": 239,
      "cached_tokens": 64,
      "reasoning_tokens": 1766,
      "cost_usd": 0.000727,
      "content_preview": "1564420181\n\n\nThe calculation 31847 \u00d7 49123 was performed using two independent decompositions to ensure accuracy.\n\n**Method 1:**  \n31847 \u00d7 49123 = 31847 \u00d7 (49000 + 123)  \n- 31847 \u00d7 49 = 1560503, then ",
      "tool_calls_count": 0,
      "error": null
    },
    {
      "model": "grok-4.20-0309-reasoning",
      "task_id": "tool_use_weather",
      "ok": true,
      "grader_msg": "called get_weather(city='paris', unit='celsius')",
      "latency_ms": 2031,
      "status": 200,
      "input_tokens": 266,
      "output_tokens": 18,
      "cached_tokens": 64,
      "reasoning_tokens": 135,
      "cost_usd": 0.00031,
      "content_preview": "",
      "tool_calls_count": 1,
      "error": null
    },
    {
      "model": "grok-4.20-0309-reasoning",
      "task_id": "code_is_prime",
      "ok": true,
      "grader_msg": "8/8 cases passed",
      "latency_ms": 9266,
      "status": 200,
      "input_tokens": 167,
      "output_tokens": 59,
      "cached_tokens": 64,
      "reasoning_tokens": 673,
      "cost_usd": 0.000289,
      "content_preview": "```python\ndef is_prime(n: int) -> bool:\n    if n <= 1: return False\n    for i in range(2, int(n**0.5) + 1):\n        if n % i == 0: return False\n    return True\n```",
      "tool_calls_count": 0,
      "error": null
    },
    {
      "model": "grok-4.3",
      "task_id": "math_multiplication",
      "ok": true,
      "grader_msg": "exact",
      "latency_ms": 11862,
      "status": 200,
      "input_tokens": 157,
      "output_tokens": 4,
      "cached_tokens": 64,
      "reasoning_tokens": 984,
      "cost_usd": 0.000139,
      "content_preview": "1564420181",
      "tool_calls_count": 0,
      "error": null
    },
    {
      "model": "grok-4.3",
      "task_id": "tool_use_weather",
      "ok": true,
      "grader_msg": "called get_weather(city='paris', unit='celsius')",
      "latency_ms": 2453,
      "status": 200,
      "input_tokens": 266,
      "output_tokens": 18,
      "cached_tokens": 64,
      "reasoning_tokens": 124,
      "cost_usd": 0.00031,
      "content_preview": "",
      "tool_calls_count": 1,
      "error": null
    },
    {
      "model": "grok-4.3",
      "task_id": "code_is_prime",
      "ok": true,
      "grader_msg": "8/8 cases passed",
      "latency_ms": 5517,
      "status": 200,
      "input_tokens": 167,
      "output_tokens": 56,
      "cached_tokens": 64,
      "reasoning_tokens": 293,
      "cost_usd": 0.000282,
      "content_preview": "def is_prime(n: int) -> bool:\n    if n <= 1:\n        return False\n    for i in range(2, int(n**0.5) + 1):\n        if n % i == 0:\n            return False\n    return True",
      "tool_calls_count": 0,
      "error": null
    },
    {
      "model": "grok-code-fast-1",
      "task_id": "math_multiplication",
      "ok": true,
      "grader_msg": "exact",
      "latency_ms": 22412,
      "status": 200,
      "input_tokens": 234,
      "output_tokens": 4,
      "cached_tokens": 197,
      "reasoning_tokens": 708,
      "cost_usd": 1.7e-05,
      "content_preview": "1564420181",
      "tool_calls_count": 0,
      "error": null
    },
    {
      "model": "grok-code-fast-1",
      "task_id": "tool_use_weather",
      "ok": true,
      "grader_msg": "called get_weather(city='paris', unit='celsius')",
      "latency_ms": 13643,
      "status": 200,
      "input_tokens": 425,
      "output_tokens": 36,
      "cached_tokens": 198,
      "reasoning_tokens": 51,
      "cost_usd": 0.000103,
      "content_preview": "",
      "tool_calls_count": 1,
      "error": null
    },
    {
      "model": "grok-code-fast-1",
      "task_id": "code_is_prime",
      "ok": true,
      "grader_msg": "8/8 cases passed",
      "latency_ms": 20860,
      "status": 200,
      "input_tokens": 244,
      "output_tokens": 62,
      "cached_tokens": 196,
      "reasoning_tokens": 360,
      "cost_usd": 0.000107,
      "content_preview": "```python\nimport math\n\ndef is_prime(n: int) -> bool:\n    if n <= 1:\n        return False\n    for i in range(2, math.isqrt(n) + 1):\n        if n % i == 0:\n            return False\n    return True\n```",
      "tool_calls_count": 0,
      "error": null
    }
  ]
}