{
  "schema_version": "engineering-cloud-replay-v1",
  "task_id": "serving-parser",
  "request_sha256": "2919895a0eb519b6f1968a8eb2f81b1dfe4d93b0c579d0142d9dbf27e2b3a6d8",
  "grader_sha256": "c83097f6069dd3e09046080b61c88aebc0922d09aab3fd5f1a03d8f04ec976f2",
  "inputs_sha256": "878ea44b1de251fa14bc9448291f1a04fc993f8aa6311115d95e424ed16d6ea7",
  "protocol_sha256": "ab806f0dc3c2e36cb8718c30f04f63bf8413baa20e5b6c1b588c23b815a49391",
  "artifacts": [
    {
      "arm": "baseline",
      "source": "DONE = \"[DONE]\"\n_FINISH = (\"length\", \"stop\")\n\n\ndef _fail(reason):\n    return {\"success\": False, \"reason\": reason}\n\n\ndef _is_count(value):\n    return type(value) is int and value >= 0\n\n\ndef evaluate(status, frames, requested_tokens):\n    if type(status) is not int or status != 200:\n        return _fail(\"http_error\")\n    if not isinstance(frames, (list, tuple)):\n        return _fail(\"malformed_frame\")\n\n    n = len(frames)\n    terminal = None\n    done_seen = False\n\n    for i, frame in enumerate(frames):\n        if isinstance(frame, str):\n            if frame == DONE:\n                if done_seen or i != n - 1:\n                    return _fail(\"malformed_frame\")\n                done_seen = True\n                continue\n            return _fail(\"malformed_frame\")\n        if not isinstance(frame, dict):\n            return _fail(\"malformed_frame\")\n        if \"error\" in frame:\n            return _fail(\"stream_error\")\n        keys = set(frame.keys())\n        if keys == {\"content\"}:\n            if not isinstance(frame[\"content\"], str):\n                return _fail(\"malformed_frame\")\n            if terminal is not None:\n                return _fail(\"malformed_frame\")\n            continue\n        if keys == {\"finish_reason\", \"completion_tokens\"}:\n            reason = frame[\"finish_reason\"]\n            count = frame[\"completion_tokens\"]\n            if not isinstance(reason, str) or reason not in _FINISH:\n                return _fail(\"malformed_frame\")\n            if not _is_count(count):\n                return _fail(\"malformed_frame\")\n            if terminal is not None:\n                return _fail(\"malformed_frame\")\n            terminal = (reason, count)\n            continue\n        return _fail(\"malformed_frame\")\n\n    if terminal is None:\n        return _fail(\"missing_terminal\")\n\n    reason, count = terminal\n    if type(requested_tokens) is not int or requested_tokens <= 0:\n        return _fail(\"token_mismatch\")\n    if reason == \"length\":\n        if count != requested_tokens:\n            return _fail(\"token_mismatch\")\n    else:\n        if count > requested_tokens:\n            return _fail(\"token_mismatch\")\n    return {\"success\": True, \"reason\": reason}\n",
      "sha256": "a671e4e1f9a9079b1cf0bb91be6a27ced248f8633ba71d491e3586c95021daab"
    },
    {
      "arm": "candidate",
      "source": "_DONE = \"[DONE]\"\n_REASONS = (\"length\", \"stop\")\n\n\ndef _fail(reason):\n    return {\"success\": False, \"reason\": reason}\n\n\ndef _is_int(value):\n    return isinstance(value, int) and not isinstance(value, bool)\n\n\ndef evaluate(status, frames, requested_tokens):\n    if not _is_int(status) or status != 200:\n        return _fail(\"http_error\")\n\n    if not isinstance(frames, (list, tuple)):\n        return _fail(\"malformed_frame\")\n\n    terminal = None\n    last_index = len(frames) - 1\n\n    for index, frame in enumerate(frames):\n        if isinstance(frame, str):\n            if frame == _DONE and index == last_index:\n                # Single optional final DONE; not a terminal by itself.\n                continue\n            return _fail(\"malformed_frame\")\n\n        if not isinstance(frame, dict):\n            return _fail(\"malformed_frame\")\n\n        if \"error\" in frame:\n            return _fail(\"stream_error\")\n\n        keys = set(frame.keys())\n\n        if keys == {\"content\"}:\n            if not isinstance(frame[\"content\"], str):\n                return _fail(\"malformed_frame\")\n            if terminal is not None:\n                return _fail(\"malformed_frame\")\n            continue\n\n        if keys == {\"finish_reason\", \"completion_tokens\"}:\n            reason = frame[\"finish_reason\"]\n            count = frame[\"completion_tokens\"]\n            if not isinstance(reason, str) or reason not in _REASONS:\n                return _fail(\"malformed_frame\")\n            if not _is_int(count) or count < 0:\n                return _fail(\"malformed_frame\")\n            if terminal is not None:\n                return _fail(\"malformed_frame\")\n            terminal = (reason, count)\n            continue\n\n        return _fail(\"malformed_frame\")\n\n    if terminal is None:\n        return _fail(\"missing_terminal\")\n\n    reason, count = terminal\n\n    if not _is_int(requested_tokens) or requested_tokens <= 0:\n        return _fail(\"token_mismatch\")\n\n    if reason == \"length\":\n        if count != requested_tokens:\n            return _fail(\"token_mismatch\")\n        return {\"success\": True, \"reason\": \"length\"}\n\n    if count > requested_tokens:\n        return _fail(\"token_mismatch\")\n    return {\"success\": True, \"reason\": \"stop\"}\n",
      "sha256": "3e3115e4e97eba8c8b998ac3bff931d7790414d31cb6e09c190e62ac4bf15219"
    }
  ]
}