{
  "model_id": "llama-3.1-8b-instant",
  "provider_id": "groq",
  "quality": {
    "accuracy": 84.3,
    "correct": 204,
    "total": 242,
    "model_id": "llama-3.1-8b-instant",
    "provider_id": "groq",
    "sample_id": "conv-43"
  },
  "retain": {
    "timestamp": "2026-02-20T13:41:43.163447+00:00",
    "model_id": "llama-3.1-8b-instant",
    "model_name": "llama-3.1-8b-instant",
    "provider_id": "groq",
    "size_gb": 0.0,
    "dataset": "locomo_3k_50",
    "concurrency": 4,
    "wall_s": 196.5937306880951,
    "summary": {
      "success": 5,
      "total": 50,
      "wall_s": 196.594,
      "avg_latency_s": 18.924,
      "throughput_rps": 0.025,
      "completion_toks_s": 29.0,
      "total_toks_s": 143.0,
      "out_in_ratio": 0.255,
      "tokens_per_fact": 51.0
    },
    "tests": [
      {
        "test_index": 1,
        "latency_s": 64.62851309776306,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: invalid_json: Expecting property name enclosed in double quotes: line 364 column 64 (char 57726) | raw: {\"facts\": [\n  {\"what\": \"James uses voice chat to communicate with his team\", \"who\": \"James\", \"why\": \"N/A\", \"entities\": [\"James\", \"team\"]},\n  {\"what\": \"James plays Apex Legends with his team\", \"who\": \"James\", \"why\": \"N/A\", \"entities\": [\"James\", \"Apex Legends\", \"team\"]},\n  {\"what\": \"James is planning "
      },
      {
        "test_index": 2,
        "latency_s": 59.181771755218506,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per minute (TPM): Limit 25000, Used 24653, Requested 4631. Please try again in 10.2816s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 3,
        "latency_s": 12.717031955718994,
        "num_facts": 19,
        "valid_json": true,
        "success": true,
        "retries": 0,
        "prompt_tokens": 4478,
        "completion_tokens": 1091,
        "error": ""
      },
      {
        "test_index": 4,
        "latency_s": 58.97960424423218,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per minute (TPM): Limit 25000, Used 24735, Requested 4680. Please try again in 10.596s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 5,
        "latency_s": 63.731456995010376,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per minute (TPM): Limit 25000, Used 24815, Requested 4569. Please try again in 10.5216s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 6,
        "latency_s": 42.84353470802307,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per minute (TPM): Limit 25000, Used 24436, Requested 4606. Please try again in 9.7008s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 7,
        "latency_s": 41.760273933410645,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per minute (TPM): Limit 25000, Used 24804, Requested 4512. Please try again in 10.3584s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 8,
        "latency_s": 47.65696883201599,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per minute (TPM): Limit 25000, Used 24494, Requested 4684. Please try again in 10.027199999s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 9,
        "latency_s": 45.87046694755554,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per minute (TPM): Limit 25000, Used 22318, Requested 4396. Please try again in 4.1136s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 10,
        "latency_s": 12.463767051696777,
        "num_facts": 30,
        "valid_json": true,
        "success": true,
        "retries": 0,
        "prompt_tokens": 4370,
        "completion_tokens": 1412,
        "error": ""
      },
      {
        "test_index": 11,
        "latency_s": 54.102970123291016,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per minute (TPM): Limit 25000, Used 24531, Requested 4525. Please try again in 9.7344s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 12,
        "latency_s": 55.04626512527466,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per minute (TPM): Limit 25000, Used 20813, Requested 4685. Please try again in 1.1952s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 13,
        "latency_s": 43.63905310630798,
        "num_facts": 23,
        "valid_json": true,
        "success": true,
        "retries": 2,
        "prompt_tokens": 4520,
        "completion_tokens": 1151,
        "error": ""
      },
      {
        "test_index": 14,
        "latency_s": 46.863219022750854,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per minute (TPM): Limit 25000, Used 24579, Requested 4636. Please try again in 10.116s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 15,
        "latency_s": 38.944591999053955,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per day (TPD): Limit 1000000, Used 997960, Requested 4701. Please try again in 3m49.9104s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 16,
        "latency_s": 12.934499025344849,
        "num_facts": 24,
        "valid_json": true,
        "success": true,
        "retries": 0,
        "prompt_tokens": 4584,
        "completion_tokens": 1130,
        "error": ""
      },
      {
        "test_index": 17,
        "latency_s": 27.610599040985107,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per day (TPD): Limit 1000000, Used 997959, Requested 4685. Please try again in 3m48.4416s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 18,
        "latency_s": 25.562236070632935,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per day (TPD): Limit 1000000, Used 997960, Requested 4641. Please try again in 3m44.7264s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 19,
        "latency_s": 12.865377187728882,
        "num_facts": 16,
        "valid_json": true,
        "success": true,
        "retries": 0,
        "prompt_tokens": 4452,
        "completion_tokens": 924,
        "error": ""
      },
      {
        "test_index": 20,
        "latency_s": 12.30669903755188,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per day (TPD): Limit 1000000, Used 997955, Requested 4634. Please try again in 3m43.6896s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 21,
        "latency_s": 0.15725302696228027,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per day (TPD): Limit 1000000, Used 997958, Requested 4685. Please try again in 3m48.3552s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 22,
        "latency_s": 0.32261204719543457,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per day (TPD): Limit 1000000, Used 997955, Requested 4585. Please try again in 3m39.456s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 23,
        "latency_s": 0.15915894508361816,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per day (TPD): Limit 1000000, Used 997957, Requested 4631. Please try again in 3m43.6032s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 24,
        "latency_s": 0.15535426139831543,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per day (TPD): Limit 1000000, Used 997956, Requested 4606. Please try again in 3m41.3568s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 25,
        "latency_s": 0.16306829452514648,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per day (TPD): Limit 1000000, Used 997955, Requested 4634. Please try again in 3m43.6896s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 26,
        "latency_s": 0.15492916107177734,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per day (TPD): Limit 1000000, Used 997955, Requested 4641. Please try again in 3m44.294399999s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 27,
        "latency_s": 0.16217708587646484,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per day (TPD): Limit 1000000, Used 997954, Requested 4371. Please try again in 3m20.88s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 28,
        "latency_s": 0.16445064544677734,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per day (TPD): Limit 1000000, Used 997953, Requested 4585. Please try again in 3m39.2832s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 29,
        "latency_s": 0.16976404190063477,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per day (TPD): Limit 1000000, Used 997953, Requested 4680. Please try again in 3m47.4912s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 30,
        "latency_s": 0.191420316696167,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per day (TPD): Limit 1000000, Used 997952, Requested 4486. Please try again in 3m30.6432s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 31,
        "latency_s": 0.16029691696166992,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per day (TPD): Limit 1000000, Used 997952, Requested 4569. Please try again in 3m37.814399999s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 32,
        "latency_s": 0.24741625785827637,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per day (TPD): Limit 1000000, Used 997951, Requested 4631. Please try again in 3m43.0848s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 33,
        "latency_s": 0.16328191757202148,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per day (TPD): Limit 1000000, Used 997951, Requested 4696. Please try again in 3m48.7008s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 34,
        "latency_s": 0.1595137119293213,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per day (TPD): Limit 1000000, Used 997951, Requested 4525. Please try again in 3m33.9264s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 35,
        "latency_s": 0.16882085800170898,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per day (TPD): Limit 1000000, Used 997950, Requested 4701. Please try again in 3m49.0464s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 36,
        "latency_s": 0.16204404830932617,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per day (TPD): Limit 1000000, Used 997949, Requested 4636. Please try again in 3m43.344s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 37,
        "latency_s": 0.17066526412963867,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per day (TPD): Limit 1000000, Used 997949, Requested 4486. Please try again in 3m30.384s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 38,
        "latency_s": 0.15846824645996094,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per day (TPD): Limit 1000000, Used 997949, Requested 4396. Please try again in 3m22.607999999s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 39,
        "latency_s": 0.2404191493988037,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per day (TPD): Limit 1000000, Used 997947, Requested 4521. Please try again in 3m33.2352s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 40,
        "latency_s": 0.2407069206237793,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per day (TPD): Limit 1000000, Used 997946, Requested 4453. Please try again in 3m27.2736s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 41,
        "latency_s": 0.15383410453796387,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per day (TPD): Limit 1000000, Used 997947, Requested 4521. Please try again in 3m33.2352s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 42,
        "latency_s": 0.23697209358215332,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per day (TPD): Limit 1000000, Used 997946, Requested 4371. Please try again in 3m20.1888s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 43,
        "latency_s": 0.16397595405578613,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per day (TPD): Limit 1000000, Used 999074, Requested 4680. Please try again in 5m24.3456s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 44,
        "latency_s": 0.2571539878845215,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per day (TPD): Limit 1000000, Used 999073, Requested 4684. Please try again in 5m24.6048s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 45,
        "latency_s": 0.24173593521118164,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per day (TPD): Limit 1000000, Used 999073, Requested 4486. Please try again in 5m7.4976s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 46,
        "latency_s": 0.2554929256439209,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per day (TPD): Limit 1000000, Used 999073, Requested 4569. Please try again in 5m14.6688s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 47,
        "latency_s": 0.16830897331237793,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per day (TPD): Limit 1000000, Used 999072, Requested 4684. Please try again in 5m24.5184s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 48,
        "latency_s": 0.16057109832763672,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per day (TPD): Limit 1000000, Used 999071, Requested 4696. Please try again in 5m25.4688s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 49,
        "latency_s": 0.24153375625610352,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per day (TPD): Limit 1000000, Used 999069, Requested 4634. Please try again in 5m19.9392s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      },
      {
        "test_index": 50,
        "latency_s": 0.2379748821258545,
        "num_facts": 0,
        "valid_json": false,
        "success": false,
        "retries": 3,
        "prompt_tokens": 0,
        "completion_tokens": 0,
        "error": "attempt 3: exception: Error code: 429 - {'error': {'message': 'Rate limit reached for model `llama-3.1-8b-instant` in organization `org_01hqv4z7ajf4yt2a7bdbdew16n` service tier `on_demand` on tokens per day (TPD): Limit 1000000, Used 999069, Requested 4479. Please try again in 5m6.5472s. Need more tokens? Visit https://groq.com/self-serve-support/ to request higher limits.', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}"
      }
    ]
  },
  "reflect": {
    "accuracy": 82.2,
    "correct": 199,
    "total": 242,
    "avg_latency_s": 61.104,
    "model_id": "llama-3.1-8b-instant",
    "provider_id": "groq",
    "sample_id": "conv-43"
  }
}