{
  "benchmark": {
    "digest_sha256": "069366e2e1fd4a22c80837544328fe34cb83ba8c1914f5cbfe8ba3bac11c0b0b",
    "id": "core",
    "ref": "core/v1",
    "task_count": 24,
    "version": "v1"
  },
  "completed_at": "2026-08-03T17:37:57.068Z",
  "configuration": {
    "endpoint": "http://127.0.0.1:8082/v1/chat/completions",
    "request": {
      "max_tokens": 256,
      "seed": 17,
      "temperature": 0,
      "top_p": 1
    },
    "run_config_sha256": "f31e0c2bf1079a2d692cb43b63a66d2d80e538f7f30ebb8fef9b87e4ba0a1165",
    "timeout_ms": 120000
  },
  "created_at": "2026-08-03T17:37:55.761Z",
  "definitions": {
    "benchmark": {
      "categories": [
        {
          "description": "Short arithmetic and proportional reasoning problems.",
          "id": "quantitative_reasoning",
          "title": "Quantitative reasoning"
        },
        {
          "description": "Deduction, ordering, and rule-following problems.",
          "id": "logical_reasoning",
          "title": "Logical reasoning"
        },
        {
          "description": "Convert small original passages into typed JSON fields.",
          "id": "information_extraction",
          "title": "Information extraction"
        },
        {
          "description": "Obey exact output, ordering, and transformation constraints.",
          "id": "instruction_following",
          "title": "Instruction following"
        }
      ],
      "description": "A compact, deterministic text benchmark for comparing model and harness behavior under identical tasks.",
      "id": "core",
      "language": "en",
      "license": "CC-BY-4.0",
      "provenance": {
        "copyrighted_sources": false,
        "origin": "Original tasks authored for fairbench.ai",
        "published": "2026-08-04"
      },
      "schema_version": "fairbench.benchmark/v1",
      "scoring": {
        "missing_and_errors": 0,
        "primary": "macro_average_by_category",
        "range": [
          0,
          1
        ]
      },
      "tasks": [
        {
          "category": "quantitative_reasoning",
          "grader": {
            "expected": "117",
            "type": "exact"
          },
          "id": "quant-01-crates",
          "prompt": "A depot receives 18 crates with 7 lamps in each crate. Nine lamps are removed for inspection. How many lamps remain? Answer with the integer only."
        },
        {
          "category": "quantitative_reasoning",
          "grader": {
            "expected": "24",
            "type": "exact"
          },
          "id": "quant-02-ratio",
          "prompt": "Red and blue tokens are in the ratio 3:5. There are 64 tokens altogether. How many are red? Answer with the integer only."
        },
        {
          "category": "quantitative_reasoning",
          "grader": {
            "expected": "36",
            "type": "exact"
          },
          "id": "quant-03-discount",
          "prompt": "A notebook costs 45 dollars before a 20 percent discount. What is the discounted price in whole dollars? Answer with the integer only."
        },
        {
          "category": "quantitative_reasoning",
          "grader": {
            "expected": "20",
            "type": "exact"
          },
          "id": "quant-04-average",
          "prompt": "Four readings are 14, 19, 22, and 25. What is their arithmetic mean? Answer with the number only."
        },
        {
          "category": "quantitative_reasoning",
          "grader": {
            "pattern": "^12:15$",
            "type": "regex"
          },
          "id": "quant-05-clock",
          "prompt": "A workshop begins at 09:40 and lasts 2 hours 35 minutes. Give the ending time in 24-hour HH:MM format."
        },
        {
          "category": "quantitative_reasoning",
          "grader": {
            "accepted": [
              "100",
              "100.0"
            ],
            "type": "exact"
          },
          "id": "quant-06-area",
          "prompt": "A rectangle is 12.5 meters long and 8 meters wide. What is its area in square meters? Answer with the number only."
        },
        {
          "category": "logical_reasoning",
          "grader": {
            "expected": "no",
            "normalize": {
              "lowercase": true
            },
            "type": "exact"
          },
          "id": "logic-01-rills",
          "prompt": "Every Rill is a Tam. No Tam is a Vek. Can any Rill be a Vek? Answer yes or no."
        },
        {
          "category": "logical_reasoning",
          "grader": {
            "expected": "Mina",
            "normalize": {
              "lowercase": true
            },
            "type": "exact"
          },
          "id": "logic-02-order",
          "prompt": "Mina arrives before Oren, and Oren arrives before Pia. Who arrives first? Answer with one name."
        },
        {
          "category": "logical_reasoning",
          "grader": {
            "expected": "down",
            "normalize": {
              "lowercase": true
            },
            "type": "exact"
          },
          "id": "logic-03-switch",
          "prompt": "A lamp is on if and only if its switch is up. The lamp is off. Is the switch up or down? Answer with one word."
        },
        {
          "category": "logical_reasoning",
          "grader": {
            "expected": "Thursday",
            "normalize": {
              "lowercase": true
            },
            "type": "exact"
          },
          "id": "logic-04-schedule",
          "prompt": "Ada works Tuesday and Thursday. Ben works Monday and Thursday. Cora works Thursday and Friday. Which weekday do all three work? Answer with the weekday only."
        },
        {
          "category": "logical_reasoning",
          "grader": {
            "flags": "i",
            "pattern": "^no$",
            "type": "regex"
          },
          "id": "logic-05-badge",
          "prompt": "Rule: every triangular badge is blue. A particular badge is not blue. Can that badge be triangular? Answer yes or no."
        },
        {
          "category": "logical_reasoning",
          "grader": {
            "expected": "L,J,K",
            "normalize": {
              "lowercase": true
            },
            "type": "exact"
          },
          "id": "logic-06-position",
          "prompt": "K is east of J. L is west of J. List the three letters from west to east, separated by commas with no spaces."
        },
        {
          "category": "information_extraction",
          "grader": {
            "allow_extra": false,
            "expected": {
              "batch": "Q7",
              "crates": 14,
              "destination": "Busan"
            },
            "normalize_strings": {
              "trim": true
            },
            "type": "json_fields"
          },
          "id": "extract-01-shipment",
          "prompt": "Shipping note: Batch Q7 contains 14 crates and is headed to Busan. Return a JSON object with fields batch, crates, and destination. crates must be a number."
        },
        {
          "category": "information_extraction",
          "grader": {
            "allow_extra": false,
            "expected": {
              "date": "2026-09-12",
              "time": "18:30",
              "title": "The Lantern Review"
            },
            "normalize_strings": {
              "trim": true
            },
            "type": "json_fields"
          },
          "id": "extract-02-event",
          "prompt": "Event card: The Lantern Review is scheduled for 2026-09-12 at 18:30. Return a JSON object with fields title, date, and time."
        },
        {
          "category": "information_extraction",
          "grader": {
            "allow_extra": false,
            "expected": {
              "scale": "C",
              "status": "stable",
              "unit": "pine-4",
              "value": 18.6
            },
            "normalize_strings": {
              "trim": true
            },
            "type": "json_fields"
          },
          "id": "extract-03-sensor",
          "prompt": "Sensor log: Unit pine-4 recorded 18.6 degrees Celsius and reported status stable. Return a JSON object with fields unit, value, scale, and status. value must be a number and scale must be C."
        },
        {
          "category": "information_extraction",
          "grader": {
            "allow_extra": false,
            "expected": {
              "grams": 240,
              "ingredient": "chickpeas",
              "vegan": true
            },
            "normalize_strings": {
              "trim": true
            },
            "type": "json_fields"
          },
          "id": "extract-04-recipe",
          "prompt": "Recipe note: Use 240 grams of chickpeas; the dish is vegan. Return a JSON object with fields ingredient, grams, and vegan. grams must be a number and vegan a boolean."
        },
        {
          "category": "information_extraction",
          "grader": {
            "allow_extra": false,
            "expected": {
              "tags": [
                "climate",
                "mystery"
              ],
              "title": "Glass Orchard",
              "year": 2031
            },
            "normalize_strings": {
              "trim": true
            },
            "type": "json_fields"
          },
          "id": "extract-05-catalog",
          "prompt": "Catalog line: Glass Orchard, published in 2031, carries the tags climate and mystery in that order. Return a JSON object with fields title, year, and tags. year must be a number and tags an array."
        },
        {
          "category": "information_extraction",
          "grader": {
            "allow_extra": false,
            "expected": {
              "advisory": null,
              "city": "Daejeon",
              "high": 29,
              "low": 18
            },
            "normalize_strings": {
              "trim": true
            },
            "type": "json_fields"
          },
          "id": "extract-06-weather",
          "prompt": "Weather brief: Daejeon expects a high of 29 and a low of 18, with no advisory. Return a JSON object with fields city, high, low, and advisory. Temperatures must be numbers and advisory must be null."
        },
        {
          "category": "instruction_following",
          "grader": {
            "expected": "fair",
            "type": "exact"
          },
          "id": "follow-01-lowercase",
          "prompt": "Write the word FAIR in lowercase. Output that word only."
        },
        {
          "category": "instruction_following",
          "grader": {
            "pattern": "^FB-207$",
            "type": "regex"
          },
          "id": "follow-02-code",
          "prompt": "Output the code FB-207 exactly, including the hyphen, and nothing else."
        },
        {
          "category": "instruction_following",
          "grader": {
            "expected": "river",
            "normalize": {
              "lowercase": true
            },
            "type": "exact"
          },
          "id": "follow-03-third-word",
          "prompt": "From the phrase 'cedar amber river silver', output only the third word."
        },
        {
          "category": "instruction_following",
          "grader": {
            "expected": "apple | kiwi | mango",
            "type": "exact"
          },
          "id": "follow-04-sort",
          "prompt": "Sort mango, apple, and kiwi alphabetically. Output the result exactly as lowercase words separated by a vertical bar with one space on each side."
        },
        {
          "category": "instruction_following",
          "grader": {
            "allow_extra": false,
            "expected": {
              "left": "SUN",
              "right": "MOON"
            },
            "type": "json_fields"
          },
          "id": "follow-05-object",
          "prompt": "Return a JSON object with exactly two fields: left must be SUN and right must be MOON."
        },
        {
          "category": "instruction_following",
          "grader": {
            "pattern": "^Quiet-Blue-Harbor$",
            "type": "regex"
          },
          "id": "follow-06-title-hyphen",
          "prompt": "Transform 'quiet blue harbor' by capitalizing the first letter of each word and joining the words with hyphens. Output only the result."
        }
      ],
      "title": "FairBench Core v1",
      "version": "v1"
    },
    "harness": {
      "description": "A single deterministic call with a short answer-only instruction and no repair pass.",
      "id": "minimal-answer",
      "license": "CC-BY-4.0",
      "messages": {
        "system": "Solve the task carefully. Return only the requested answer, with no explanation, preamble, or markdown fences.",
        "user_template": "{{prompt}}"
      },
      "request": {
        "max_tokens": 256,
        "retries": 0,
        "seed": 17,
        "temperature": 0,
        "top_p": 1
      },
      "response": {
        "mode": "text"
      },
      "schema_version": "fairbench.harness/v1",
      "title": "Minimal answer",
      "version": "v1"
    }
  },
  "harness": {
    "digest_sha256": "2fa1144b02c1552114479a4b59e323bd2d92c17598cb9565619834fdf16a9c30",
    "id": "minimal-answer",
    "ref": "minimal-answer/v1",
    "version": "v1"
  },
  "integrity": {
    "algorithm": "sha256",
    "bundle_sha256": "fe52823d5b398c467fb09388204e0ca2fc883abecb05915be9cd582959f78003"
  },
  "items": [
    {
      "category": "quantitative_reasoning",
      "extraction": {
        "answer": "10",
        "error": null,
        "ok": true
      },
      "finished_at": "2026-08-03T17:37:55.825Z",
      "grade": {
        "details": {
          "actual": "10",
          "expected": [
            "117"
          ]
        },
        "grader": "exact",
        "passed": false,
        "score": 0
      },
      "input_sha256": "57ed14599e2359421bbac363091903f53c72950036c4b37c06162722acff4455",
      "latency_ms": 50.724,
      "request": {
        "body": {
          "max_tokens": 256,
          "messages": [
            {
              "content": "Solve the task carefully. Return only the requested answer, with no explanation, preamble, or markdown fences.",
              "role": "system"
            },
            {
              "content": "A depot receives 18 crates with 7 lamps in each crate. Nine lamps are removed for inspection. How many lamps remain? Answer with the integer only.",
              "role": "user"
            }
          ],
          "model": "smollm2-1.7b-instruct",
          "seed": 17,
          "stream": false,
          "temperature": 0,
          "top_p": 1
        },
        "request_sha256": "6c5afe568c1dd18de4be3048d0c06ff34e43651764e2a9dbace47395a5893edc"
      },
      "response": {
        "content": "10",
        "finish_reason": "stop",
        "output_sha256": "4a44dc15364204a80fe80e9039455cc1608281820fe2b24f1e5233ade6af1dd5",
        "usage": {
          "completion_tokens": 3,
          "prompt_tokens": 73,
          "prompt_tokens_details": {
            "cached_tokens": 0
          },
          "total_tokens": 76
        }
      },
      "status": "completed",
      "task_id": "quant-01-crates",
      "task_sha256": "9730729cd6a00efa4440b418acf4d53631a2c161aefb08b42c76ebb12e269e58"
    },
    {
      "category": "quantitative_reasoning",
      "extraction": {
        "answer": "24",
        "error": null,
        "ok": true
      },
      "finished_at": "2026-08-03T17:37:55.849Z",
      "grade": {
        "details": {
          "actual": "24",
          "expected": [
            "24"
          ]
        },
        "grader": "exact",
        "passed": true,
        "score": 1
      },
      "input_sha256": "e86a1c406e7af17044ea608e06d672be91bd48cefa8a49d3a3cc14c9e7414ea8",
      "latency_ms": 22.961,
      "request": {
        "body": {
          "max_tokens": 256,
          "messages": [
            {
              "content": "Solve the task carefully. Return only the requested answer, with no explanation, preamble, or markdown fences.",
              "role": "system"
            },
            {
              "content": "Red and blue tokens are in the ratio 3:5. There are 64 tokens altogether. How many are red? Answer with the integer only.",
              "role": "user"
            }
          ],
          "model": "smollm2-1.7b-instruct",
          "seed": 17,
          "stream": false,
          "temperature": 0,
          "top_p": 1
        },
        "request_sha256": "b3e13167e24bc0b5c4941144e539d055e8a7b66da4f70b9d0a857436a24a01e7"
      },
      "response": {
        "content": "24",
        "finish_reason": "stop",
        "output_sha256": "c2356069e9d1e79ca924378153cfbbfb4d4416b1f99d41a2940bfdb66c5319db",
        "usage": {
          "completion_tokens": 3,
          "prompt_tokens": 70,
          "prompt_tokens_details": {
            "cached_tokens": 32
          },
          "total_tokens": 73
        }
      },
      "status": "completed",
      "task_id": "quant-02-ratio",
      "task_sha256": "e8fcb0e0c71a22b1b84b1f1fd6d0bec97880be273d3ba6231f4a555934b912b6"
    },
    {
      "category": "quantitative_reasoning",
      "extraction": {
        "answer": "30",
        "error": null,
        "ok": true
      },
      "finished_at": "2026-08-03T17:37:55.874Z",
      "grade": {
        "details": {
          "actual": "30",
          "expected": [
            "36"
          ]
        },
        "grader": "exact",
        "passed": false,
        "score": 0
      },
      "input_sha256": "818463be6c0bc7fc37005ae9be33e4cf321a30def3cf1e07f63d379034b0e92a",
      "latency_ms": 22.367,
      "request": {
        "body": {
          "max_tokens": 256,
          "messages": [
            {
              "content": "Solve the task carefully. Return only the requested answer, with no explanation, preamble, or markdown fences.",
              "role": "system"
            },
            {
              "content": "A notebook costs 45 dollars before a 20 percent discount. What is the discounted price in whole dollars? Answer with the integer only.",
              "role": "user"
            }
          ],
          "model": "smollm2-1.7b-instruct",
          "seed": 17,
          "stream": false,
          "temperature": 0,
          "top_p": 1
        },
        "request_sha256": "f2fb8d316b68235f636b2b0e514d708bc2c7f4491ef96d5b53ad4f095c72fc94"
      },
      "response": {
        "content": "30",
        "finish_reason": "stop",
        "output_sha256": "624b60c58c9d8bfb6ff1886c2fd605d2adeb6ea4da576068201b6c6958ce93f4",
        "usage": {
          "completion_tokens": 3,
          "prompt_tokens": 68,
          "prompt_tokens_details": {
            "cached_tokens": 32
          },
          "total_tokens": 71
        }
      },
      "status": "completed",
      "task_id": "quant-03-discount",
      "task_sha256": "228c893b627fda46c495aabd0bc6196723b1d524f645a8e1d58cd473e2c19563"
    },
    {
      "category": "quantitative_reasoning",
      "extraction": {
        "answer": "19",
        "error": null,
        "ok": true
      },
      "finished_at": "2026-08-03T17:37:55.900Z",
      "grade": {
        "details": {
          "actual": "19",
          "expected": [
            "20"
          ]
        },
        "grader": "exact",
        "passed": false,
        "score": 0
      },
      "input_sha256": "7f3d7b0f5f793aede5f2f00ce8a08f97c4c8816797de18a236c6e213f52bd53d",
      "latency_ms": 22.083,
      "request": {
        "body": {
          "max_tokens": 256,
          "messages": [
            {
              "content": "Solve the task carefully. Return only the requested answer, with no explanation, preamble, or markdown fences.",
              "role": "system"
            },
            {
              "content": "Four readings are 14, 19, 22, and 25. What is their arithmetic mean? Answer with the number only.",
              "role": "user"
            }
          ],
          "model": "smollm2-1.7b-instruct",
          "seed": 17,
          "stream": false,
          "temperature": 0,
          "top_p": 1
        },
        "request_sha256": "b9cec2584b46e6095938c8a5ac7c941041d60fbd09d7fcdf443c77466de99c08"
      },
      "response": {
        "content": "19",
        "finish_reason": "stop",
        "output_sha256": "9400f1b21cb527d7fa3d3eabba93557a18ebe7a2ca4e471cfe5e4c5b4ca7f767",
        "usage": {
          "completion_tokens": 3,
          "prompt_tokens": 70,
          "prompt_tokens_details": {
            "cached_tokens": 32
          },
          "total_tokens": 73
        }
      },
      "status": "completed",
      "task_id": "quant-04-average",
      "task_sha256": "e0e7477f72c95cd177fb6b35699a80f755d238d1087111fa439cf566b9db271a"
    },
    {
      "category": "quantitative_reasoning",
      "extraction": {
        "answer": "17:15",
        "error": null,
        "ok": true
      },
      "finished_at": "2026-08-03T17:37:55.935Z",
      "grade": {
        "details": {
          "actual": "17:15",
          "flags": "",
          "pattern": "^12:15$"
        },
        "grader": "regex",
        "passed": false,
        "score": 0
      },
      "input_sha256": "7e50bd62e398aea77d2f863c0124bf7eccd1db23c9ae5603280a451ea07c8bb8",
      "latency_ms": 33.319,
      "request": {
        "body": {
          "max_tokens": 256,
          "messages": [
            {
              "content": "Solve the task carefully. Return only the requested answer, with no explanation, preamble, or markdown fences.",
              "role": "system"
            },
            {
              "content": "A workshop begins at 09:40 and lasts 2 hours 35 minutes. Give the ending time in 24-hour HH:MM format.",
              "role": "user"
            }
          ],
          "model": "smollm2-1.7b-instruct",
          "seed": 17,
          "stream": false,
          "temperature": 0,
          "top_p": 1
        },
        "request_sha256": "151438e25d4c4066f25f7420a852ca0b524c9b255fb12cdabe7ccb14928ec676"
      },
      "response": {
        "content": "17:15",
        "finish_reason": "stop",
        "output_sha256": "721cdb2d16ce85d87ccb28314d8b6aba2373518b2ae3fa8a8a4df26d517b1f40",
        "usage": {
          "completion_tokens": 6,
          "prompt_tokens": 74,
          "prompt_tokens_details": {
            "cached_tokens": 33
          },
          "total_tokens": 80
        }
      },
      "status": "completed",
      "task_id": "quant-05-clock",
      "task_sha256": "474f95f13ce271ca26f2ada90bbd5a455bf68589106f33083e6fc4efc869b251"
    },
    {
      "category": "quantitative_reasoning",
      "extraction": {
        "answer": "100",
        "error": null,
        "ok": true
      },
      "finished_at": "2026-08-03T17:37:55.963Z",
      "grade": {
        "details": {
          "actual": "100",
          "expected": [
            "100",
            "100.0"
          ]
        },
        "grader": "exact",
        "passed": true,
        "score": 1
      },
      "input_sha256": "19a432843dd1d47d569a89094d8ce986b3b8c41b53426f8799871c169a62072b",
      "latency_ms": 25.503,
      "request": {
        "body": {
          "max_tokens": 256,
          "messages": [
            {
              "content": "Solve the task carefully. Return only the requested answer, with no explanation, preamble, or markdown fences.",
              "role": "system"
            },
            {
              "content": "A rectangle is 12.5 meters long and 8 meters wide. What is its area in square meters? Answer with the number only.",
              "role": "user"
            }
          ],
          "model": "smollm2-1.7b-instruct",
          "seed": 17,
          "stream": false,
          "temperature": 0,
          "top_p": 1
        },
        "request_sha256": "55c4bf3380850a6f8b4338541b41ff04d6b32f7fa468ab7ecf600c17d575b64b"
      },
      "response": {
        "content": "100",
        "finish_reason": "stop",
        "output_sha256": "ad57366865126e55649ecb23ae1d48887544976efea46a48eb5d85a6eeb4d306",
        "usage": {
          "completion_tokens": 4,
          "prompt_tokens": 68,
          "prompt_tokens_details": {
            "cached_tokens": 33
          },
          "total_tokens": 72
        }
      },
      "status": "completed",
      "task_id": "quant-06-area",
      "task_sha256": "f92d2e52aee0de11c118157e115821e703c5adfe962f59270732d0ff4ad3ad59"
    },
    {
      "category": "logical_reasoning",
      "extraction": {
        "answer": "no",
        "error": null,
        "ok": true
      },
      "finished_at": "2026-08-03T17:37:55.984Z",
      "grade": {
        "details": {
          "actual": "no",
          "expected": [
            "no"
          ]
        },
        "grader": "exact",
        "passed": true,
        "score": 1
      },
      "input_sha256": "ed2385dcce176c8f48337ef333c5334d96e8fe2c2b6bb0fc58559afd3958d353",
      "latency_ms": 18.961,
      "request": {
        "body": {
          "max_tokens": 256,
          "messages": [
            {
              "content": "Solve the task carefully. Return only the requested answer, with no explanation, preamble, or markdown fences.",
              "role": "system"
            },
            {
              "content": "Every Rill is a Tam. No Tam is a Vek. Can any Rill be a Vek? Answer yes or no.",
              "role": "user"
            }
          ],
          "model": "smollm2-1.7b-instruct",
          "seed": 17,
          "stream": false,
          "temperature": 0,
          "top_p": 1
        },
        "request_sha256": "31aca6fe083204202cc17063d8497ef5bb95b5eee01e35e1c74321aa6a6bcbe2"
      },
      "response": {
        "content": "no",
        "finish_reason": "stop",
        "output_sha256": "9390298f3fb0c5b160498935d79cb139aef28e1c47358b4bbba61862b9c26e59",
        "usage": {
          "completion_tokens": 2,
          "prompt_tokens": 66,
          "prompt_tokens_details": {
            "cached_tokens": 32
          },
          "total_tokens": 68
        }
      },
      "status": "completed",
      "task_id": "logic-01-rills",
      "task_sha256": "79cc8af6aa290cfcf2515158edefff8a647a1111261fd2c5c080cf7ece5c15d4"
    },
    {
      "category": "logical_reasoning",
      "extraction": {
        "answer": "Mina",
        "error": null,
        "ok": true
      },
      "finished_at": "2026-08-03T17:37:56.006Z",
      "grade": {
        "details": {
          "actual": "mina",
          "expected": [
            "mina"
          ]
        },
        "grader": "exact",
        "passed": true,
        "score": 1
      },
      "input_sha256": "887f5986ba43fcae476e0c89c929cc360be12a2256e419e1ae8509e10a5c3d25",
      "latency_ms": 20.513,
      "request": {
        "body": {
          "max_tokens": 256,
          "messages": [
            {
              "content": "Solve the task carefully. Return only the requested answer, with no explanation, preamble, or markdown fences.",
              "role": "system"
            },
            {
              "content": "Mina arrives before Oren, and Oren arrives before Pia. Who arrives first? Answer with one name.",
              "role": "user"
            }
          ],
          "model": "smollm2-1.7b-instruct",
          "seed": 17,
          "stream": false,
          "temperature": 0,
          "top_p": 1
        },
        "request_sha256": "a1f3ad8ec37e087f4998886732b8152193643ab76f45dcfd963a39dc7fb1b1e5"
      },
      "response": {
        "content": "Mina",
        "finish_reason": "stop",
        "output_sha256": "830449e3d5d3711c531223ac4f532a42fd100ef5691bcb8210b36d9a9df010d9",
        "usage": {
          "completion_tokens": 3,
          "prompt_tokens": 61,
          "prompt_tokens_details": {
            "cached_tokens": 32
          },
          "total_tokens": 64
        }
      },
      "status": "completed",
      "task_id": "logic-02-order",
      "task_sha256": "64930ea4695bdb6aede625c5d52f58eb81871f70dcc3e334f43ce07f45acd2e8"
    },
    {
      "category": "logical_reasoning",
      "extraction": {
        "answer": "down",
        "error": null,
        "ok": true
      },
      "finished_at": "2026-08-03T17:37:56.026Z",
      "grade": {
        "details": {
          "actual": "down",
          "expected": [
            "down"
          ]
        },
        "grader": "exact",
        "passed": true,
        "score": 1
      },
      "input_sha256": "86d62455d0ec0028f643ece6bc7d219736d0c445eeb84478e520b1d34c387118",
      "latency_ms": 17.589,
      "request": {
        "body": {
          "max_tokens": 256,
          "messages": [
            {
              "content": "Solve the task carefully. Return only the requested answer, with no explanation, preamble, or markdown fences.",
              "role": "system"
            },
            {
              "content": "A lamp is on if and only if its switch is up. The lamp is off. Is the switch up or down? Answer with one word.",
              "role": "user"
            }
          ],
          "model": "smollm2-1.7b-instruct",
          "seed": 17,
          "stream": false,
          "temperature": 0,
          "top_p": 1
        },
        "request_sha256": "a60bf6667f162dafa9fc0aaec7f8694157b79bc60a0dd92ad98c4b128d2532e0"
      },
      "response": {
        "content": "down",
        "finish_reason": "stop",
        "output_sha256": "908aec4512d80ff4fefb1970899091e9de8e734b36b8fdb7678e77dc092f6959",
        "usage": {
          "completion_tokens": 2,
          "prompt_tokens": 68,
          "prompt_tokens_details": {
            "cached_tokens": 32
          },
          "total_tokens": 70
        }
      },
      "status": "completed",
      "task_id": "logic-03-switch",
      "task_sha256": "f5c9dd7808be2d48ac76e08c901c4113bc95bb9cc57cd10dabc4c3e732e7686b"
    },
    {
      "category": "logical_reasoning",
      "extraction": {
        "answer": "Monday",
        "error": null,
        "ok": true
      },
      "finished_at": "2026-08-03T17:37:56.048Z",
      "grade": {
        "details": {
          "actual": "monday",
          "expected": [
            "thursday"
          ]
        },
        "grader": "exact",
        "passed": false,
        "score": 0
      },
      "input_sha256": "4b5f954cc0bb4b82b1611a9fc1bd89fc5aabef172b7029570dd8120c9e4e098d",
      "latency_ms": 19.045,
      "request": {
        "body": {
          "max_tokens": 256,
          "messages": [
            {
              "content": "Solve the task carefully. Return only the requested answer, with no explanation, preamble, or markdown fences.",
              "role": "system"
            },
            {
              "content": "Ada works Tuesday and Thursday. Ben works Monday and Thursday. Cora works Thursday and Friday. Which weekday do all three work? Answer with the weekday only.",
              "role": "user"
            }
          ],
          "model": "smollm2-1.7b-instruct",
          "seed": 17,
          "stream": false,
          "temperature": 0,
          "top_p": 1
        },
        "request_sha256": "67d541e09539468852128fdb1ac521aed9604d8802bf2c0e3e4f87112397491d"
      },
      "response": {
        "content": "Monday",
        "finish_reason": "stop",
        "output_sha256": "6a00dfc1dc867e8454c2c8856e1512d9bf02a76710e3411c0972aec886c76c61",
        "usage": {
          "completion_tokens": 2,
          "prompt_tokens": 73,
          "prompt_tokens_details": {
            "cached_tokens": 32
          },
          "total_tokens": 75
        }
      },
      "status": "completed",
      "task_id": "logic-04-schedule",
      "task_sha256": "8a35a7e7c3b000a3a61bdcc6d80be5cf9a4991916c944ec493d2590c322721e6"
    },
    {
      "category": "logical_reasoning",
      "extraction": {
        "answer": "no",
        "error": null,
        "ok": true
      },
      "finished_at": "2026-08-03T17:37:56.067Z",
      "grade": {
        "details": {
          "actual": "no",
          "flags": "i",
          "pattern": "^no$"
        },
        "grader": "regex",
        "passed": true,
        "score": 1
      },
      "input_sha256": "a89096bb1b64b85311c3c5b3c28cced674ad3b0f602648b47b00b0e3cbd670f7",
      "latency_ms": 17.145,
      "request": {
        "body": {
          "max_tokens": 256,
          "messages": [
            {
              "content": "Solve the task carefully. Return only the requested answer, with no explanation, preamble, or markdown fences.",
              "role": "system"
            },
            {
              "content": "Rule: every triangular badge is blue. A particular badge is not blue. Can that badge be triangular? Answer yes or no.",
              "role": "user"
            }
          ],
          "model": "smollm2-1.7b-instruct",
          "seed": 17,
          "stream": false,
          "temperature": 0,
          "top_p": 1
        },
        "request_sha256": "c170ca7830da82452bf14466082edc7a41cb1ad8821520e6046f5c5505603a56"
      },
      "response": {
        "content": "no",
        "finish_reason": "stop",
        "output_sha256": "9390298f3fb0c5b160498935d79cb139aef28e1c47358b4bbba61862b9c26e59",
        "usage": {
          "completion_tokens": 2,
          "prompt_tokens": 64,
          "prompt_tokens_details": {
            "cached_tokens": 32
          },
          "total_tokens": 66
        }
      },
      "status": "completed",
      "task_id": "logic-05-badge",
      "task_sha256": "c9930bc53ab6d6f7ff34793473d06f2b95557e824e39c91c8f7c59b6812562fd"
    },
    {
      "category": "logical_reasoning",
      "extraction": {
        "answer": "jlk",
        "error": null,
        "ok": true
      },
      "finished_at": "2026-08-03T17:37:56.094Z",
      "grade": {
        "details": {
          "actual": "jlk",
          "expected": [
            "l,j,k"
          ]
        },
        "grader": "exact",
        "passed": false,
        "score": 0
      },
      "input_sha256": "c7567d53511d94e2d9abf6db1871d5bcc22e80a4533b7730fb0e19654c961954",
      "latency_ms": 25.312,
      "request": {
        "body": {
          "max_tokens": 256,
          "messages": [
            {
              "content": "Solve the task carefully. Return only the requested answer, with no explanation, preamble, or markdown fences.",
              "role": "system"
            },
            {
              "content": "K is east of J. L is west of J. List the three letters from west to east, separated by commas with no spaces.",
              "role": "user"
            }
          ],
          "model": "smollm2-1.7b-instruct",
          "seed": 17,
          "stream": false,
          "temperature": 0,
          "top_p": 1
        },
        "request_sha256": "964e144b4e7456430fdb4ced5e819ed091603cef3ddb4289c28aed30e239fc19"
      },
      "response": {
        "content": "jlk",
        "finish_reason": "stop",
        "output_sha256": "6e4a6ecb26817bd7609d74f5bae585a6a02b23d0737167f71c6bdc0c76ee147f",
        "usage": {
          "completion_tokens": 4,
          "prompt_tokens": 66,
          "prompt_tokens_details": {
            "cached_tokens": 32
          },
          "total_tokens": 70
        }
      },
      "status": "completed",
      "task_id": "logic-06-position",
      "task_sha256": "00aa5bc0568b606d9a3201c8c4a95cb74f1200d078dbeee2a37f1b687ce3aa8f"
    },
    {
      "category": "information_extraction",
      "extraction": {
        "answer": "{\n  \"batch\": \"Q7\",\n  \"crates\": 14,\n  \"destination\": \"Busan\"\n}",
        "error": null,
        "ok": true
      },
      "finished_at": "2026-08-03T17:37:56.209Z",
      "grade": {
        "details": {
          "matched": 3,
          "mismatches": [],
          "total": 3
        },
        "grader": "json_fields",
        "passed": true,
        "score": 1
      },
      "input_sha256": "949965a08e5f301dff120b349502146918236829d3f79e5954e90f93e4382620",
      "latency_ms": 112.332,
      "request": {
        "body": {
          "max_tokens": 256,
          "messages": [
            {
              "content": "Solve the task carefully. Return only the requested answer, with no explanation, preamble, or markdown fences.",
              "role": "system"
            },
            {
              "content": "Shipping note: Batch Q7 contains 14 crates and is headed to Busan. Return a JSON object with fields batch, crates, and destination. crates must be a number.",
              "role": "user"
            }
          ],
          "model": "smollm2-1.7b-instruct",
          "seed": 17,
          "stream": false,
          "temperature": 0,
          "top_p": 1
        },
        "request_sha256": "509d2e540686208d4e918ca061a9c76a693f5154e0c2fd1b6039b57e82edec55"
      },
      "response": {
        "content": "{\n  \"batch\": \"Q7\",\n  \"crates\": 14,\n  \"destination\": \"Busan\"\n}",
        "finish_reason": "stop",
        "output_sha256": "34a004d5e04d9763d2dbb172daec5ac051ad41363689cc1ccf56e828a6637ff6",
        "usage": {
          "completion_tokens": 29,
          "prompt_tokens": 79,
          "prompt_tokens_details": {
            "cached_tokens": 32
          },
          "total_tokens": 108
        }
      },
      "status": "completed",
      "task_id": "extract-01-shipment",
      "task_sha256": "e42b0a3338c9f80325b1cd4cacc5e432881081216f07913819369fc19a0b7d80"
    },
    {
      "category": "information_extraction",
      "extraction": {
        "answer": "{\"title\":\"The Lantern Review\",\"date\":\"2026-09-12\",\"time\":\"18:30\"}",
        "error": null,
        "ok": true
      },
      "finished_at": "2026-08-03T17:37:56.328Z",
      "grade": {
        "details": {
          "matched": 3,
          "mismatches": [],
          "total": 3
        },
        "grader": "json_fields",
        "passed": true,
        "score": 1
      },
      "input_sha256": "5a0772b1217500be8a0e8a53489b5aae07f77d4b93c6ce3008ea04a2ea17b09d",
      "latency_ms": 116.736,
      "request": {
        "body": {
          "max_tokens": 256,
          "messages": [
            {
              "content": "Solve the task carefully. Return only the requested answer, with no explanation, preamble, or markdown fences.",
              "role": "system"
            },
            {
              "content": "Event card: The Lantern Review is scheduled for 2026-09-12 at 18:30. Return a JSON object with fields title, date, and time.",
              "role": "user"
            }
          ],
          "model": "smollm2-1.7b-instruct",
          "seed": 17,
          "stream": false,
          "temperature": 0,
          "top_p": 1
        },
        "request_sha256": "0438e5dc81bbe36b66d347d0ee8e45220e7c4c65cc9feaa27c497ae7ce297638"
      },
      "response": {
        "content": "{\"title\":\"The Lantern Review\",\"date\":\"2026-09-12\",\"time\":\"18:30\"}",
        "finish_reason": "stop",
        "output_sha256": "30ade0dcd1f9b739d2b5f4a0319c603ad28ff807b087bba83b7ccd37c1947a69",
        "usage": {
          "completion_tokens": 30,
          "prompt_tokens": 80,
          "prompt_tokens_details": {
            "cached_tokens": 32
          },
          "total_tokens": 110
        }
      },
      "status": "completed",
      "task_id": "extract-02-event",
      "task_sha256": "a7eea073ac8572680dd812d18539207fe8f347eedbe04a1d93fe01376f522ce0"
    },
    {
      "category": "information_extraction",
      "extraction": {
        "answer": "{\n  \"unit\": \"Unit\",\n  \"value\": 18.6,\n  \"scale\": \"C\",\n  \"status\": \"stable\"\n}",
        "error": null,
        "ok": true
      },
      "finished_at": "2026-08-03T17:37:56.465Z",
      "grade": {
        "details": {
          "matched": 3,
          "mismatches": [
            "unit"
          ],
          "total": 4
        },
        "grader": "json_fields",
        "passed": false,
        "score": 0.75
      },
      "input_sha256": "0d918c4b6321def1d0cbcb528bfd2a4e5cb837224f829838c73b7e8461420099",
      "latency_ms": 133.929,
      "request": {
        "body": {
          "max_tokens": 256,
          "messages": [
            {
              "content": "Solve the task carefully. Return only the requested answer, with no explanation, preamble, or markdown fences.",
              "role": "system"
            },
            {
              "content": "Sensor log: Unit pine-4 recorded 18.6 degrees Celsius and reported status stable. Return a JSON object with fields unit, value, scale, and status. value must be a number and scale must be C.",
              "role": "user"
            }
          ],
          "model": "smollm2-1.7b-instruct",
          "seed": 17,
          "stream": false,
          "temperature": 0,
          "top_p": 1
        },
        "request_sha256": "7d43bb40428fabd7bdda81959962862c7945dac868f6ea302b08ba5fd31f1e73"
      },
      "response": {
        "content": "{\n  \"unit\": \"Unit\",\n  \"value\": 18.6,\n  \"scale\": \"C\",\n  \"status\": \"stable\"\n}",
        "finish_reason": "stop",
        "output_sha256": "c93ddf039b7d12217a29fdb5c8f740317555ddd1d11899e7acfe3906b0adce9a",
        "usage": {
          "completion_tokens": 35,
          "prompt_tokens": 84,
          "prompt_tokens_details": {
            "cached_tokens": 32
          },
          "total_tokens": 119
        }
      },
      "status": "completed",
      "task_id": "extract-03-sensor",
      "task_sha256": "676dc7d16096f751c807e3fbe508640de1ae2d79d898f3525e9cf5887dcae127"
    },
    {
      "category": "information_extraction",
      "extraction": {
        "answer": "{\n  \"ingredient\": \"240 grams\",\n  \"grams\": 240,\n  \"vegan\": true\n}",
        "error": null,
        "ok": true
      },
      "finished_at": "2026-08-03T17:37:56.589Z",
      "grade": {
        "details": {
          "matched": 2,
          "mismatches": [
            "ingredient"
          ],
          "total": 3
        },
        "grader": "json_fields",
        "passed": false,
        "score": 0.6666666666666666
      },
      "input_sha256": "f045f31f38e6aaee5b5608bd6a3f320b5ae830d03b0f52868657ffa48476d3bc",
      "latency_ms": 120.264,
      "request": {
        "body": {
          "max_tokens": 256,
          "messages": [
            {
              "content": "Solve the task carefully. Return only the requested answer, with no explanation, preamble, or markdown fences.",
              "role": "system"
            },
            {
              "content": "Recipe note: Use 240 grams of chickpeas; the dish is vegan. Return a JSON object with fields ingredient, grams, and vegan. grams must be a number and vegan a boolean.",
              "role": "user"
            }
          ],
          "model": "smollm2-1.7b-instruct",
          "seed": 17,
          "stream": false,
          "temperature": 0,
          "top_p": 1
        },
        "request_sha256": "a05eef616daed0f38c7541a2a1897b124939a0c8e1c15fac34ee89572807e9a8"
      },
      "response": {
        "content": "{\n  \"ingredient\": \"240 grams\",\n  \"grams\": 240,\n  \"vegan\": true\n}",
        "finish_reason": "stop",
        "output_sha256": "acbac5060e9d6489685248b0e9b14b4fcbe173d9a60761279fcda2d16dbafedc",
        "usage": {
          "completion_tokens": 31,
          "prompt_tokens": 78,
          "prompt_tokens_details": {
            "cached_tokens": 32
          },
          "total_tokens": 109
        }
      },
      "status": "completed",
      "task_id": "extract-04-recipe",
      "task_sha256": "a6112ccb2cdf94064773d5c87cb92a0eb07c8209af6ea53dc6911ee9603d6a44"
    },
    {
      "category": "information_extraction",
      "extraction": {
        "answer": "{\n  \"title\": \"Glass Orchard\",\n  \"year\": 2031,\n  \"tags\": [\"climate\", \"mystery\"]\n}",
        "error": null,
        "ok": true
      },
      "finished_at": "2026-08-03T17:37:56.725Z",
      "grade": {
        "details": {
          "matched": 3,
          "mismatches": [],
          "total": 3
        },
        "grader": "json_fields",
        "passed": true,
        "score": 1
      },
      "input_sha256": "dfc6547940005d5f926691bd3ae2c5ead7096d148b17b9136b5ce9da4ea4224b",
      "latency_ms": 133.472,
      "request": {
        "body": {
          "max_tokens": 256,
          "messages": [
            {
              "content": "Solve the task carefully. Return only the requested answer, with no explanation, preamble, or markdown fences.",
              "role": "system"
            },
            {
              "content": "Catalog line: Glass Orchard, published in 2031, carries the tags climate and mystery in that order. Return a JSON object with fields title, year, and tags. year must be a number and tags an array.",
              "role": "user"
            }
          ],
          "model": "smollm2-1.7b-instruct",
          "seed": 17,
          "stream": false,
          "temperature": 0,
          "top_p": 1
        },
        "request_sha256": "59eddffb93ad15586ff096bbb76944e5b52d4dec67f6cca2e5d2fd4dbc76cbfb"
      },
      "response": {
        "content": "{\n  \"title\": \"Glass Orchard\",\n  \"year\": 2031,\n  \"tags\": [\"climate\", \"mystery\"]\n}",
        "finish_reason": "stop",
        "output_sha256": "665ca1e4206064ff22f44402034f5e111f2163f79e0ef42a2c5fde98b7d2d4c2",
        "usage": {
          "completion_tokens": 35,
          "prompt_tokens": 87,
          "prompt_tokens_details": {
            "cached_tokens": 32
          },
          "total_tokens": 122
        }
      },
      "status": "completed",
      "task_id": "extract-05-catalog",
      "task_sha256": "e2d49553c9847d7e7e702f6d88cfa0ab71d74e555d591d8cfd858c63c3428048"
    },
    {
      "category": "information_extraction",
      "extraction": {
        "answer": "{\n  \"city\": \"Daejeon\",\n  \"high\": 29,\n  \"low\": 18\n}",
        "error": null,
        "ok": true
      },
      "finished_at": "2026-08-03T17:37:56.840Z",
      "grade": {
        "details": {
          "matched": 3,
          "mismatches": [
            "advisory",
            "<extra-or-missing-root-fields>"
          ],
          "total": 4
        },
        "grader": "json_fields",
        "passed": false,
        "score": 0
      },
      "input_sha256": "a354272b45f595b5de6f24035f2215be1e93ab857f433048bfe7fab81a9f921d",
      "latency_ms": 112.725,
      "request": {
        "body": {
          "max_tokens": 256,
          "messages": [
            {
              "content": "Solve the task carefully. Return only the requested answer, with no explanation, preamble, or markdown fences.",
              "role": "system"
            },
            {
              "content": "Weather brief: Daejeon expects a high of 29 and a low of 18, with no advisory. Return a JSON object with fields city, high, low, and advisory. Temperatures must be numbers and advisory must be null.",
              "role": "user"
            }
          ],
          "model": "smollm2-1.7b-instruct",
          "seed": 17,
          "stream": false,
          "temperature": 0,
          "top_p": 1
        },
        "request_sha256": "68b029998b4ab7cec3bef6fd0342686e8686b2bd806d003dd42bc2c04e005447"
      },
      "response": {
        "content": "{\n  \"city\": \"Daejeon\",\n  \"high\": 29,\n  \"low\": 18\n}",
        "finish_reason": "stop",
        "output_sha256": "50541c53feed7742cba2279fa8b90a719c077b98b6d9d23090b952e1e94d5baf",
        "usage": {
          "completion_tokens": 29,
          "prompt_tokens": 89,
          "prompt_tokens_details": {
            "cached_tokens": 32
          },
          "total_tokens": 118
        }
      },
      "status": "completed",
      "task_id": "extract-06-weather",
      "task_sha256": "ed45e146ac9eba5e7e74e6ac9d2d937b56e8fb8a2820411b4444f14aea9092e4"
    },
    {
      "category": "instruction_following",
      "extraction": {
        "answer": "faire",
        "error": null,
        "ok": true
      },
      "finished_at": "2026-08-03T17:37:56.864Z",
      "grade": {
        "details": {
          "actual": "faire",
          "expected": [
            "fair"
          ]
        },
        "grader": "exact",
        "passed": false,
        "score": 0
      },
      "input_sha256": "0e9de6ef7276e328dd1d1fcbba6aa3fda7e25d4150af56fd0803a72fdb85cf3d",
      "latency_ms": 21.484,
      "request": {
        "body": {
          "max_tokens": 256,
          "messages": [
            {
              "content": "Solve the task carefully. Return only the requested answer, with no explanation, preamble, or markdown fences.",
              "role": "system"
            },
            {
              "content": "Write the word FAIR in lowercase. Output that word only.",
              "role": "user"
            }
          ],
          "model": "smollm2-1.7b-instruct",
          "seed": 17,
          "stream": false,
          "temperature": 0,
          "top_p": 1
        },
        "request_sha256": "8365948a83270a372e75cceefa6624d2fc3fb7edee56c8d587216e6dbbeb1fc6"
      },
      "response": {
        "content": "faire",
        "finish_reason": "stop",
        "output_sha256": "7f802a7788c545c6ae5863a48f0edef80602e3fd201368f175a2cb6ccc70ba4a",
        "usage": {
          "completion_tokens": 3,
          "prompt_tokens": 51,
          "prompt_tokens_details": {
            "cached_tokens": 32
          },
          "total_tokens": 54
        }
      },
      "status": "completed",
      "task_id": "follow-01-lowercase",
      "task_sha256": "7a84f67ed3687518b2106af84e4210e31eeb14baa340dc6c66ebe4797129d550"
    },
    {
      "category": "instruction_following",
      "extraction": {
        "answer": "fb-207",
        "error": null,
        "ok": true
      },
      "finished_at": "2026-08-03T17:37:56.897Z",
      "grade": {
        "details": {
          "actual": "fb-207",
          "flags": "",
          "pattern": "^FB-207$"
        },
        "grader": "regex",
        "passed": false,
        "score": 0
      },
      "input_sha256": "5cf4f859cc67feac7697d3f94177b0de5c381a3d325a26d8afb8757f6406fee0",
      "latency_ms": 29.699,
      "request": {
        "body": {
          "max_tokens": 256,
          "messages": [
            {
              "content": "Solve the task carefully. Return only the requested answer, with no explanation, preamble, or markdown fences.",
              "role": "system"
            },
            {
              "content": "Output the code FB-207 exactly, including the hyphen, and nothing else.",
              "role": "user"
            }
          ],
          "model": "smollm2-1.7b-instruct",
          "seed": 17,
          "stream": false,
          "temperature": 0,
          "top_p": 1
        },
        "request_sha256": "f72047ee7ea32d75a8a29fbdca447f91bc71674b45d5fd9757230e42be228fde"
      },
      "response": {
        "content": "fb-207",
        "finish_reason": "stop",
        "output_sha256": "1ad96ec7647ed0eb65c5b528b4516bdd920c09f50b78e1328c92d1d83292b102",
        "usage": {
          "completion_tokens": 6,
          "prompt_tokens": 57,
          "prompt_tokens_details": {
            "cached_tokens": 32
          },
          "total_tokens": 63
        }
      },
      "status": "completed",
      "task_id": "follow-02-code",
      "task_sha256": "ffc3be529529e7623e4db32bb0bd7c67180ae30d55c83c6dcad88f4b0fc70e7e"
    },
    {
      "category": "instruction_following",
      "extraction": {
        "answer": "river",
        "error": null,
        "ok": true
      },
      "finished_at": "2026-08-03T17:37:56.916Z",
      "grade": {
        "details": {
          "actual": "river",
          "expected": [
            "river"
          ]
        },
        "grader": "exact",
        "passed": true,
        "score": 1
      },
      "input_sha256": "0ffff63da206b3899a63e9570aad9120cae24928e5cec18d2f442750d4aac485",
      "latency_ms": 16.282,
      "request": {
        "body": {
          "max_tokens": 256,
          "messages": [
            {
              "content": "Solve the task carefully. Return only the requested answer, with no explanation, preamble, or markdown fences.",
              "role": "system"
            },
            {
              "content": "From the phrase 'cedar amber river silver', output only the third word.",
              "role": "user"
            }
          ],
          "model": "smollm2-1.7b-instruct",
          "seed": 17,
          "stream": false,
          "temperature": 0,
          "top_p": 1
        },
        "request_sha256": "0165cc43dc1dc62a2e14db93ac9377862db55ce8dbef73446511dc16c553fd2f"
      },
      "response": {
        "content": "river",
        "finish_reason": "stop",
        "output_sha256": "5f5a8ed8f139be6df7cf8f4a91d3e4649961a98c82fd521c18d6f292b031a319",
        "usage": {
          "completion_tokens": 2,
          "prompt_tokens": 54,
          "prompt_tokens_details": {
            "cached_tokens": 32
          },
          "total_tokens": 56
        }
      },
      "status": "completed",
      "task_id": "follow-03-third-word",
      "task_sha256": "2b0876f3b0dd769ff28c731b106a118d352b379c93c86efaaaf910743ab97ae1"
    },
    {
      "category": "instruction_following",
      "extraction": {
        "answer": "mango apple kiwi",
        "error": null,
        "ok": true
      },
      "finished_at": "2026-08-03T17:37:56.946Z",
      "grade": {
        "details": {
          "actual": "mango apple kiwi",
          "expected": [
            "apple | kiwi | mango"
          ]
        },
        "grader": "exact",
        "passed": false,
        "score": 0
      },
      "input_sha256": "5ad5fe99cc776c7a6bd94c469c17df729d877f56e2fbb97db120a149e419d2fa",
      "latency_ms": 27.515,
      "request": {
        "body": {
          "max_tokens": 256,
          "messages": [
            {
              "content": "Solve the task carefully. Return only the requested answer, with no explanation, preamble, or markdown fences.",
              "role": "system"
            },
            {
              "content": "Sort mango, apple, and kiwi alphabetically. Output the result exactly as lowercase words separated by a vertical bar with one space on each side.",
              "role": "user"
            }
          ],
          "model": "smollm2-1.7b-instruct",
          "seed": 17,
          "stream": false,
          "temperature": 0,
          "top_p": 1
        },
        "request_sha256": "cca34acdada59a1947229c4fc16c2b471ce16b43ce1e0cc2762013cd2c8c3b85"
      },
      "response": {
        "content": "mango apple kiwi",
        "finish_reason": "stop",
        "output_sha256": "b523daab6de276fb7975729967e73e8da41e4112ee99fabb9dfa5b76dcec9d3a",
        "usage": {
          "completion_tokens": 5,
          "prompt_tokens": 67,
          "prompt_tokens_details": {
            "cached_tokens": 32
          },
          "total_tokens": 72
        }
      },
      "status": "completed",
      "task_id": "follow-04-sort",
      "task_sha256": "6848935608ca8b99fb3ff676a9f8d19f735498b64d58ab32881554f462d968b2"
    },
    {
      "category": "instruction_following",
      "extraction": {
        "answer": "{\n  \"left\": \"SUN\",\n  \"right\": \"MOON\"\n}",
        "error": null,
        "ok": true
      },
      "finished_at": "2026-08-03T17:37:57.028Z",
      "grade": {
        "details": {
          "matched": 2,
          "mismatches": [],
          "total": 2
        },
        "grader": "json_fields",
        "passed": true,
        "score": 1
      },
      "input_sha256": "24b88c139429763b6eb9803da8db48b4799c02609f9493a0f13c9b2f3ee6b1d5",
      "latency_ms": 79.095,
      "request": {
        "body": {
          "max_tokens": 256,
          "messages": [
            {
              "content": "Solve the task carefully. Return only the requested answer, with no explanation, preamble, or markdown fences.",
              "role": "system"
            },
            {
              "content": "Return a JSON object with exactly two fields: left must be SUN and right must be MOON.",
              "role": "user"
            }
          ],
          "model": "smollm2-1.7b-instruct",
          "seed": 17,
          "stream": false,
          "temperature": 0,
          "top_p": 1
        },
        "request_sha256": "0426efd37174752d71f4e4c89f01712e801578ac7bde9636a5a2c041660c52de"
      },
      "response": {
        "content": "{\n  \"left\": \"SUN\",\n  \"right\": \"MOON\"\n}",
        "finish_reason": "stop",
        "output_sha256": "8981e5910a2fe0932ddef1b2c7201499e24905f9c085db3e762863845daf7950",
        "usage": {
          "completion_tokens": 20,
          "prompt_tokens": 59,
          "prompt_tokens_details": {
            "cached_tokens": 32
          },
          "total_tokens": 79
        }
      },
      "status": "completed",
      "task_id": "follow-05-object",
      "task_sha256": "1fd705ba2b7ceab4c3072d18d994b5b7bc222d1f7033d861a77ad18af25d3bbc"
    },
    {
      "category": "instruction_following",
      "extraction": {
        "answer": "quiet-blue-harbor",
        "error": null,
        "ok": true
      },
      "finished_at": "2026-08-03T17:37:57.066Z",
      "grade": {
        "details": {
          "actual": "quiet-blue-harbor",
          "flags": "",
          "pattern": "^Quiet-Blue-Harbor$"
        },
        "grader": "regex",
        "passed": false,
        "score": 0
      },
      "input_sha256": "26f24b024c3e7fa8e8ef32131a37c2db3064982c536e44f8cf74d2401409261b",
      "latency_ms": 35.482,
      "request": {
        "body": {
          "max_tokens": 256,
          "messages": [
            {
              "content": "Solve the task carefully. Return only the requested answer, with no explanation, preamble, or markdown fences.",
              "role": "system"
            },
            {
              "content": "Transform 'quiet blue harbor' by capitalizing the first letter of each word and joining the words with hyphens. Output only the result.",
              "role": "user"
            }
          ],
          "model": "smollm2-1.7b-instruct",
          "seed": 17,
          "stream": false,
          "temperature": 0,
          "top_p": 1
        },
        "request_sha256": "a507aff8673cd2d1ab7f239385c0c1c16e7e925e65d963fd575536747c0ceeb6"
      },
      "response": {
        "content": "quiet-blue-harbor",
        "finish_reason": "stop",
        "output_sha256": "8c85c221ca5e31b334f06d5c0ab3f2e15e5d76d009d414e8d24609618c61fd77",
        "usage": {
          "completion_tokens": 7,
          "prompt_tokens": 66,
          "prompt_tokens_details": {
            "cached_tokens": 32
          },
          "total_tokens": 73
        }
      },
      "status": "completed",
      "task_id": "follow-06-title-hyphen",
      "task_sha256": "e11e4ce8d0473a8aea714292a3b51cf5bea107de56908592440fcf0655d1ba91"
    }
  ],
  "model": {
    "artifact_digest": "decd2598bc2c8ed08c19adc3c8fdd461ee19ed5708679d1c54ef54a5a30d4f33",
    "descriptor_sha256": "0fc0707c6228744fcbb3425156f28540612f175eafea572666aed36ff93e3e03",
    "id": "smollm2-1.7b-instruct",
    "quantization": "GGUF-Q4_K_M",
    "revision": "Q4_K_M"
  },
  "provenance": {
    "local_execution": true,
    "secrets_recorded": false,
    "tier": "reference"
  },
  "run_id": "44ecef7e-1446-43d5-9aed-27da65f304a0",
  "runner": {
    "architecture": "arm64",
    "git_commit": null,
    "name": "fairbench-runner",
    "node": "v24.13.0",
    "platform": "darwin",
    "source_sha256": "b4eb95ad9a05e2ba899d49f56d691422110f9327115944a5206ea3a169bd09d5",
    "version": "0.1.0"
  },
  "schema_version": "fairbench.result/v1",
  "status": "complete",
  "summary": {
    "category_scores": {
      "information_extraction": 0.736111,
      "instruction_following": 0.333333,
      "logical_reasoning": 0.666667,
      "quantitative_reasoning": 0.333333
    },
    "completed_count": 24,
    "confidence_interval_95": [
      0.322917,
      0.711806
    ],
    "confidence_method": "deterministic task bootstrap (10000 resamples)",
    "coverage": 1,
    "error_count": 0,
    "mean_latency_ms": 51.439,
    "parse_failure_count": 0,
    "parse_success_rate": 1,
    "passed_count": 11,
    "reported_task_count": 24,
    "score": 0.517361,
    "score_unit": "fraction",
    "task_count": 24,
    "token_usage": {
      "completion_tokens": 269,
      "prompt_tokens": 1672,
      "total_tokens": 1941
    },
    "total_latency_ms": 1234.537
  }
}