{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-a1", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 5271, "output_tokens": 344, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-a1", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 4968, "output_tokens": 336, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-a1", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 5038, "output_tokens": 312, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-a1", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 5468, "output_tokens": 349, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-a1", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 13445, "output_tokens": 573, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-a1", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 6941, "output_tokens": 170, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 619, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-a1", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 10898, "output_tokens": 183, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2365, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-a1", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2891, "output_tokens": 83, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-a1", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 11194, "output_tokens": 168, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2217, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-a1", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 15923, "output_tokens": 359, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2372, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-a1", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2932, "output_tokens": 126, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-a1", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 5978, "output_tokens": 255, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-a1", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2710, "output_tokens": 193, "cache_read_tokens": 14623, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-a1", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2218, "output_tokens": 176, "cache_read_tokens": 43829, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-a1", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 4151, "output_tokens": 160, "cache_read_tokens": 4624, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-a1", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 12206, "output_tokens": 324, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 619, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-a1", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 3704, "output_tokens": 84, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-a1", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 27256, "output_tokens": 365, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 3962, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-a1", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 3454, "output_tokens": 151, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-a1", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 30911, "output_tokens": 620, "cache_read_tokens": 3525, "cache_write_tokens": 0, "retrieved_tokens": 3134, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-a1", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 3319, "output_tokens": 91, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-a1", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2545, "output_tokens": 97, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-a1", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2998, "output_tokens": 95, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-a1", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2584, "output_tokens": 62, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-a1", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 6588, "output_tokens": 284, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c1", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 3233, "output_tokens": 368, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c1", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 3037, "output_tokens": 366, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c1", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 3184, "output_tokens": 377, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c1", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 3373, "output_tokens": 379, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c1", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 7852, "output_tokens": 977, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c1", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2179, "output_tokens": 160, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c1", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 0.0, "cr_score": 0.5, "sub_em_score": null, "input_tokens": 9520, "output_tokens": 246, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2369, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c1", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2028, "output_tokens": 136, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c1", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2195, "output_tokens": 165, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c1", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 18677, "output_tokens": 539, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 4730, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c1", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1811, "output_tokens": 143, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c1", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.9, "rf_score": 1.0, "cr_score": 0.95, "sub_em_score": null, "input_tokens": 4526, "output_tokens": 217, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c1", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1436, "output_tokens": 183, "cache_read_tokens": 14623, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c1", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1090, "output_tokens": 189, "cache_read_tokens": 43829, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c1", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1923, "output_tokens": 311, "cache_read_tokens": 4624, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c1", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 6423, "output_tokens": 257, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 619, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c1", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 24946, "output_tokens": 411, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 3457, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c1", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 25258, "output_tokens": 397, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 3983, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c1", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2607, "output_tokens": 146, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c1", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 11509, "output_tokens": 396, "cache_read_tokens": 2356, "cache_write_tokens": 0, "retrieved_tokens": 2644, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c1", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2139, "output_tokens": 129, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c1", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 0.5, "cr_score": 0.75, "sub_em_score": null, "input_tokens": 1799, "output_tokens": 131, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c1", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 0.5, "cr_score": 0.75, "sub_em_score": null, "input_tokens": 2156, "output_tokens": 162, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c1", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 0.5, "cr_score": 0.75, "sub_em_score": null, "input_tokens": 1827, "output_tokens": 101, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c1", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 4247, "output_tokens": 430, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c2", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 3621, "output_tokens": 368, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c2", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 3427, "output_tokens": 344, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c2", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 3581, "output_tokens": 376, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c2", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 3772, "output_tokens": 387, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c2", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 9067, "output_tokens": 1011, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c2", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2031, "output_tokens": 116, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c2", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2174, "output_tokens": 87, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c2", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2162, "output_tokens": 95, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c2", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2189, "output_tokens": 127, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c2", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 11909, "output_tokens": 281, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2301, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c2", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1982, "output_tokens": 118, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c2", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 4787, "output_tokens": 201, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c2", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1639, "output_tokens": 144, "cache_read_tokens": 14623, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c2", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1290, "output_tokens": 164, "cache_read_tokens": 43829, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c2", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2232, "output_tokens": 241, "cache_read_tokens": 4624, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c2", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2696, "output_tokens": 132, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c2", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2793, "output_tokens": 130, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c2", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2420, "output_tokens": 94, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c2", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2592, "output_tokens": 125, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c2", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 11106, "output_tokens": 318, "cache_read_tokens": 2350, "cache_write_tokens": 0, "retrieved_tokens": 2331, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c2", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2280, "output_tokens": 119, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c2", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 0.0, "cr_score": 0.5, "sub_em_score": null, "input_tokens": 1950, "output_tokens": 97, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c2", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.5, "rf_score": 0.0, "cr_score": 0.25, "sub_em_score": null, "input_tokens": 2338, "output_tokens": 123, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c2", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 0.0, "cr_score": 0.5, "sub_em_score": null, "input_tokens": 1948, "output_tokens": 87, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c2", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 0.0, "cr_score": 0.5, "sub_em_score": null, "input_tokens": 4719, "output_tokens": 395, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c3", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 4014, "output_tokens": 343, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c3", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 3796, "output_tokens": 353, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c3", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 3977, "output_tokens": 323, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c3", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 4190, "output_tokens": 349, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c3", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 10145, "output_tokens": 1024, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c3", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2187, "output_tokens": 159, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c3", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2286, "output_tokens": 142, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c3", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2324, "output_tokens": 165, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c3", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2341, "output_tokens": 160, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c3", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 13322, "output_tokens": 503, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2365, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c3", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2169, "output_tokens": 197, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c3", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 5013, "output_tokens": 265, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c3", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1808, "output_tokens": 242, "cache_read_tokens": 14623, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c3", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1479, "output_tokens": 200, "cache_read_tokens": 43829, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c3", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.8, "rf_score": 1.0, "cr_score": 0.9, "sub_em_score": null, "input_tokens": 2520, "output_tokens": 480, "cache_read_tokens": 4624, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c3", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2853, "output_tokens": 190, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c3", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2966, "output_tokens": 200, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c3", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2539, "output_tokens": 129, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c3", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2760, "output_tokens": 192, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c3", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 9609, "output_tokens": 588, "cache_read_tokens": 2350, "cache_write_tokens": 0, "retrieved_tokens": 1917, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c3", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2434, "output_tokens": 183, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c3", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 2082, "output_tokens": 135, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c3", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 2498, "output_tokens": 151, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c3", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 2058, "output_tokens": 153, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c3", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 5153, "output_tokens": 425, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c4", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 4402, "output_tokens": 381, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c4", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 4185, "output_tokens": 379, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c4", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 4316, "output_tokens": 366, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c4", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 4576, "output_tokens": 380, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c4", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 11128, "output_tokens": 1117, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c4", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2371, "output_tokens": 189, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c4", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2453, "output_tokens": 186, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c4", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2501, "output_tokens": 171, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c4", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2549, "output_tokens": 185, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c4", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 3975, "output_tokens": 418, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c4", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2394, "output_tokens": 237, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c4", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 5303, "output_tokens": 304, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c4", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 0.5, "cr_score": 0.75, "sub_em_score": null, "input_tokens": 2075, "output_tokens": 284, "cache_read_tokens": 14623, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c4", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1701, "output_tokens": 250, "cache_read_tokens": 43829, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c4", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.8, "rf_score": 1.0, "cr_score": 0.9, "sub_em_score": null, "input_tokens": 3022, "output_tokens": 552, "cache_read_tokens": 4624, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c4", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 3068, "output_tokens": 211, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c4", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 3207, "output_tokens": 221, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c4", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2672, "output_tokens": 163, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c4", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2977, "output_tokens": 226, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c4", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 11885, "output_tokens": 689, "cache_read_tokens": 2350, "cache_write_tokens": 0, "retrieved_tokens": 2405, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c4", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2684, "output_tokens": 216, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c4", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 0.5, "cr_score": 0.75, "sub_em_score": null, "input_tokens": 2232, "output_tokens": 156, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c4", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 2667, "output_tokens": 143, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c4", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 0.0, "cr_score": 0.5, "sub_em_score": null, "input_tokens": 2236, "output_tokens": 193, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c4", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 0.0, "cr_score": 0.5, "sub_em_score": null, "input_tokens": 5591, "output_tokens": 485, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c5", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 4843, "output_tokens": 366, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c5", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 4585, "output_tokens": 369, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c5", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 4668, "output_tokens": 356, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c5", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 4994, "output_tokens": 394, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c5", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 12309, "output_tokens": 1079, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c5", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2581, "output_tokens": 194, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c5", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2660, "output_tokens": 149, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c5", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2709, "output_tokens": 168, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c5", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2755, "output_tokens": 196, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c5", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 4330, "output_tokens": 406, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c5", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2652, "output_tokens": 266, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c5", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 5628, "output_tokens": 336, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c5", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2380, "output_tokens": 308, "cache_read_tokens": 14623, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c5", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1938, "output_tokens": 288, "cache_read_tokens": 43829, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c5", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 3599, "output_tokens": 562, "cache_read_tokens": 4624, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c5", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 3300, "output_tokens": 197, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c5", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 3444, "output_tokens": 243, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c5", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2856, "output_tokens": 183, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c5", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 3224, "output_tokens": 216, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c5", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 3489, "output_tokens": 505, "cache_read_tokens": 1178, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c5", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2972, "output_tokens": 236, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c5", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 2394, "output_tokens": 137, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c5", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 2848, "output_tokens": 136, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c5", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 2450, "output_tokens": 120, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-c5", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 6098, "output_tokens": 472, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r1", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 48, "output_tokens": 244, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r1", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 48, "output_tokens": 235, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r1", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 48, "output_tokens": 248, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r1", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 48, "output_tokens": 289, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r1", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 72, "output_tokens": 446, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r1", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 3178, "output_tokens": 219, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 619, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r1", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 7093, "output_tokens": 172, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2386, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r1", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 7024, "output_tokens": 237, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2315, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r1", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 7083, "output_tokens": 204, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2355, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r1", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 8229, "output_tokens": 401, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2389, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r1", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 574, "output_tokens": 94, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r1", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 3164, "output_tokens": 111, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r1", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 16, "output_tokens": 103, "cache_read_tokens": 14623, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r1", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 16, "output_tokens": 85, "cache_read_tokens": 0, "cache_write_tokens": 43829, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r1", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 16, "output_tokens": 206, "cache_read_tokens": 4624, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r1", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 3780, "output_tokens": 268, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 619, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r1", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 6816, "output_tokens": 300, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2014, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r1", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 8089, "output_tokens": 240, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2600, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r1", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 8054, "output_tokens": 303, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2600, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r1", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 5749, "output_tokens": 368, "cache_read_tokens": 1178, "cache_write_tokens": 1178, "retrieved_tokens": 2052, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r1", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 2786, "output_tokens": 251, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 619, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r1", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.5, "rf_score": 0.0, "cr_score": 0.25, "sub_em_score": 0.0, "input_tokens": 6160, "output_tokens": 160, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2240, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r1", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.15, "rf_score": 0.0, "cr_score": 0.075, "sub_em_score": 0.0, "input_tokens": 6139, "output_tokens": 277, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2238, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r1", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.5, "rf_score": 0.0, "cr_score": 0.25, "sub_em_score": 0.0, "input_tokens": 6158, "output_tokens": 153, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2246, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r1", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.15, "rf_score": 0.0, "cr_score": 0.075, "sub_em_score": 0.0, "input_tokens": 7027, "output_tokens": 457, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2240, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r2", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 305, "output_tokens": 287, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r2", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 296, "output_tokens": 264, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r2", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 309, "output_tokens": 292, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r2", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 350, "output_tokens": 324, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r2", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 538, "output_tokens": 676, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r2", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 3450, "output_tokens": 188, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 619, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r2", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 7164, "output_tokens": 189, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2277, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r2", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 7346, "output_tokens": 226, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2338, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r2", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 7250, "output_tokens": 205, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2371, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r2", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 14941, "output_tokens": 301, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 4601, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r2", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 681, "output_tokens": 149, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r2", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 3288, "output_tokens": 133, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r2", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 132, "output_tokens": 110, "cache_read_tokens": 14623, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r2", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 114, "output_tokens": 116, "cache_read_tokens": 43829, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r2", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 242, "output_tokens": 136, "cache_read_tokens": 4624, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r2", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 4093, "output_tokens": 264, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 619, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r2", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 7072, "output_tokens": 237, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1923, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r2", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 8987, "output_tokens": 207, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2849, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r2", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 9093, "output_tokens": 222, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2849, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r2", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 7840, "output_tokens": 257, "cache_read_tokens": 2356, "cache_write_tokens": 0, "retrieved_tokens": 2629, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r2", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 3154, "output_tokens": 224, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 619, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r2", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 6343, "output_tokens": 209, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2240, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r2", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 6570, "output_tokens": 227, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2238, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r2", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 6330, "output_tokens": 201, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2246, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r2", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 7812, "output_tokens": 458, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2240, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r3", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.15, "rf_score": 0.0, "cr_score": 0.075, "sub_em_score": 0.0, "input_tokens": 605, "output_tokens": 366, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r3", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.15, "rf_score": 0.0, "cr_score": 0.075, "sub_em_score": 0.0, "input_tokens": 573, "output_tokens": 295, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r3", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.15, "rf_score": 0.0, "cr_score": 0.075, "sub_em_score": 0.0, "input_tokens": 622, "output_tokens": 326, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r3", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 687, "output_tokens": 370, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r3", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 1219, "output_tokens": 908, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r3", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 3660, "output_tokens": 213, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 619, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r3", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 7356, "output_tokens": 181, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2329, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r3", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 7523, "output_tokens": 217, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2307, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r3", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 7474, "output_tokens": 225, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2319, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r3", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 8912, "output_tokens": 238, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2324, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r3", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 843, "output_tokens": 126, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r3", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 3433, "output_tokens": 127, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r3", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 255, "output_tokens": 118, "cache_read_tokens": 14623, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r3", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 243, "output_tokens": 107, "cache_read_tokens": 43829, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r3", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 415, "output_tokens": 120, "cache_read_tokens": 4624, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r3", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 4446, "output_tokens": 271, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 619, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r3", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 6743, "output_tokens": 201, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1771, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r3", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 9063, "output_tokens": 211, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2866, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r3", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 8501, "output_tokens": 226, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2454, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r3", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 5942, "output_tokens": 249, "cache_read_tokens": 2356, "cache_write_tokens": 0, "retrieved_tokens": 1815, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r3", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 3468, "output_tokens": 247, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 619, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r3", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 6628, "output_tokens": 183, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2240, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r3", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 6884, "output_tokens": 195, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2238, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r3", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 6573, "output_tokens": 158, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2246, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r3", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 8582, "output_tokens": 402, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2240, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r4", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 982, "output_tokens": 361, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r4", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 865, "output_tokens": 347, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r4", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 936, "output_tokens": 369, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r4", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 1075, "output_tokens": 395, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r4", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 2162, "output_tokens": 981, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r4", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 3916, "output_tokens": 198, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 619, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r4", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 7580, "output_tokens": 319, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2364, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r4", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 7854, "output_tokens": 283, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2356, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r4", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 7724, "output_tokens": 223, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2266, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r4", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 9491, "output_tokens": 490, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2350, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r4", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 980, "output_tokens": 109, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r4", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 3571, "output_tokens": 210, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r4", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 384, "output_tokens": 188, "cache_read_tokens": 14623, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r4", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 361, "output_tokens": 109, "cache_read_tokens": 43829, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r4", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 552, "output_tokens": 305, "cache_read_tokens": 4624, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r4", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 4826, "output_tokens": 234, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 619, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r4", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 10440, "output_tokens": 356, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 3305, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r4", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 11637, "output_tokens": 296, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 3809, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r4", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.5, "rf_score": 0.3, "cr_score": 0.4, "sub_em_score": 0.0, "input_tokens": 12357, "output_tokens": 297, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 3999, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r4", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 9962, "output_tokens": 445, "cache_read_tokens": 2356, "cache_write_tokens": 0, "retrieved_tokens": 3214, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r4", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 3874, "output_tokens": 219, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 619, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r4", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.5, "rf_score": 0.3, "cr_score": 0.4, "sub_em_score": 0.0, "input_tokens": 6858, "output_tokens": 217, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2240, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r4", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.5, "rf_score": 0.3, "cr_score": 0.4, "sub_em_score": 0.0, "input_tokens": 7071, "output_tokens": 265, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2238, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r4", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.5, "rf_score": 0.3, "cr_score": 0.4, "sub_em_score": 0.0, "input_tokens": 6751, "output_tokens": 250, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2246, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r4", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.5, "rf_score": 0.3, "cr_score": 0.4, "sub_em_score": 0.0, "input_tokens": 9180, "output_tokens": 554, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2240, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r5", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 1352, "output_tokens": 352, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r5", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 1216, "output_tokens": 338, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r5", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.15, "rf_score": 0.0, "cr_score": 0.075, "sub_em_score": 0.0, "input_tokens": 1309, "output_tokens": 332, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r5", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 1479, "output_tokens": 358, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r5", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 3178, "output_tokens": 917, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r5", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 4131, "output_tokens": 197, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 619, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r5", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 7959, "output_tokens": 199, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2297, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r5", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 8171, "output_tokens": 296, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2354, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r5", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 19935, "output_tokens": 439, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 4556, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r5", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 10220, "output_tokens": 368, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2297, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r5", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 1121, "output_tokens": 120, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r5", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 3836, "output_tokens": 182, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r5", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 581, "output_tokens": 168, "cache_read_tokens": 14623, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r5", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 479, "output_tokens": 113, "cache_read_tokens": 43829, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r5", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 840, "output_tokens": 204, "cache_read_tokens": 4624, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r5", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 5107, "output_tokens": 227, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 619, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r5", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 9781, "output_tokens": 345, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2669, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r5", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 11440, "output_tokens": 192, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 3554, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r5", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 11487, "output_tokens": 205, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 3554, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r5", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 8947, "output_tokens": 421, "cache_read_tokens": 2356, "cache_write_tokens": 0, "retrieved_tokens": 2668, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r5", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 4170, "output_tokens": 190, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 619, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r5", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 7116, "output_tokens": 265, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2240, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r5", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 7445, "output_tokens": 399, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2238, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r5", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 7102, "output_tokens": 334, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2246, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r5", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 10117, "output_tokens": 751, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2240, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r6", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.3333333333333333, "input_tokens": 1715, "output_tokens": 316, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r6", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 1565, "output_tokens": 312, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r6", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.15, "rf_score": 0.0, "cr_score": 0.075, "sub_em_score": 0.3333333333333333, "input_tokens": 1670, "output_tokens": 320, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r6", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.3333333333333333, "input_tokens": 1848, "output_tokens": 331, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r6", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 4116, "output_tokens": 637, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r6", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.6666666666666666, "input_tokens": 4318, "output_tokens": 165, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 619, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r6", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.6666666666666666, "input_tokens": 8501, "output_tokens": 144, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2351, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r6", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.6666666666666666, "input_tokens": 8465, "output_tokens": 149, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2283, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r6", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.6666666666666666, "input_tokens": 8349, "output_tokens": 181, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2232, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r6", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.6666666666666666, "input_tokens": 10789, "output_tokens": 238, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2353, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r6", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.6666666666666666, "input_tokens": 1262, "output_tokens": 94, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r6", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.6666666666666666, "input_tokens": 3939, "output_tokens": 91, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r6", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.6666666666666666, "input_tokens": 760, "output_tokens": 135, "cache_read_tokens": 14623, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r6", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.6666666666666666, "input_tokens": 600, "output_tokens": 107, "cache_read_tokens": 43829, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r6", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.6666666666666666, "input_tokens": 1049, "output_tokens": 129, "cache_read_tokens": 4624, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r6", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.6666666666666666, "input_tokens": 4955, "output_tokens": 168, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 619, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r6", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.6666666666666666, "input_tokens": 10551, "output_tokens": 192, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 3138, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r6", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.6666666666666666, "input_tokens": 10734, "output_tokens": 160, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 3138, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r6", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.6666666666666666, "input_tokens": 10800, "output_tokens": 172, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 3138, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r6", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.6666666666666666, "input_tokens": 10710, "output_tokens": 221, "cache_read_tokens": 2356, "cache_write_tokens": 0, "retrieved_tokens": 3138, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r6", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.6666666666666666, "input_tokens": 4412, "output_tokens": 185, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 619, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r6", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 1607, "output_tokens": 100, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r6", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 1626, "output_tokens": 97, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r6", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 1481, "output_tokens": 63, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r6", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 11290, "output_tokens": 382, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2240, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r7", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 2047, "output_tokens": 375, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r7", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 1886, "output_tokens": 337, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r7", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.15, "rf_score": 0.0, "cr_score": 0.075, "sub_em_score": 0.0, "input_tokens": 2013, "output_tokens": 365, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r7", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 2195, "output_tokens": 374, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r7", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 4842, "output_tokens": 971, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r7", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 4483, "output_tokens": 204, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 619, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r7", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 8668, "output_tokens": 202, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2384, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r7", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 8642, "output_tokens": 209, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2294, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r7", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 8544, "output_tokens": 202, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2348, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r7", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 11063, "output_tokens": 252, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2382, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r7", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 1372, "output_tokens": 129, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r7", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 4063, "output_tokens": 160, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r7", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 911, "output_tokens": 149, "cache_read_tokens": 14623, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r7", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 720, "output_tokens": 111, "cache_read_tokens": 43829, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r7", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 1182, "output_tokens": 189, "cache_read_tokens": 4624, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r7", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 5546, "output_tokens": 235, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 619, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r7", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 9421, "output_tokens": 165, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2537, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r7", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 12798, "output_tokens": 227, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 3981, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r7", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 12913, "output_tokens": 210, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 3979, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r7", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 8286, "output_tokens": 252, "cache_read_tokens": 2356, "cache_write_tokens": 0, "retrieved_tokens": 2086, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r7", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 4654, "output_tokens": 242, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 619, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r7", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.85, "rf_score": 0.4, "cr_score": 0.625, "sub_em_score": 0.0, "input_tokens": 1723, "output_tokens": 124, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r7", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 8139, "output_tokens": 270, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2238, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r7", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 7774, "output_tokens": 136, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2246, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r7", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 11901, "output_tokens": 540, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2240, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r8", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.1, "rf_score": 0.0, "cr_score": 0.05, "sub_em_score": 0.0, "input_tokens": 2432, "output_tokens": 371, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r8", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.15, "rf_score": 0.0, "cr_score": 0.075, "sub_em_score": 0.0, "input_tokens": 2233, "output_tokens": 380, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r8", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.15, "rf_score": 0.0, "cr_score": 0.075, "sub_em_score": 0.0, "input_tokens": 2402, "output_tokens": 370, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r8", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.15, "rf_score": 0.0, "cr_score": 0.075, "sub_em_score": 0.0, "input_tokens": 2560, "output_tokens": 383, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r8", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.15, "rf_score": 0.0, "cr_score": 0.075, "sub_em_score": 0.0, "input_tokens": 5896, "output_tokens": 940, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r8", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.5, "input_tokens": 4746, "output_tokens": 184, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 619, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r8", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 8851, "output_tokens": 178, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2321, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r8", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 8846, "output_tokens": 203, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2254, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r8", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 8729, "output_tokens": 203, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2262, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r8", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 11353, "output_tokens": 241, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2324, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r8", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 1511, "output_tokens": 143, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r8", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 4204, "output_tokens": 136, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r8", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 1070, "output_tokens": 179, "cache_read_tokens": 14623, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r8", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 841, "output_tokens": 120, "cache_read_tokens": 43829, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r8", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 1382, "output_tokens": 175, "cache_read_tokens": 4624, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r8", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 5860, "output_tokens": 229, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 619, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r8", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 11505, "output_tokens": 241, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2982, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r8", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 12357, "output_tokens": 196, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 3737, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r8", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 12601, "output_tokens": 225, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 3737, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r8", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 11038, "output_tokens": 241, "cache_read_tokens": 2356, "cache_write_tokens": 0, "retrieved_tokens": 2995, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r8", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 5014, "output_tokens": 230, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 619, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r8", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 7847, "output_tokens": 160, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2240, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r8", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 1923, "output_tokens": 94, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r8", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 1659, "output_tokens": 70, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r8", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 11397, "output_tokens": 359, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2240, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r9", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 2814, "output_tokens": 394, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r9", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 2639, "output_tokens": 376, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r9", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 2795, "output_tokens": 389, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r9", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 2954, "output_tokens": 398, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r9", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 6829, "output_tokens": 1121, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r9", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 0.5, "cr_score": 0.75, "sub_em_score": 0.0, "input_tokens": 2016, "output_tokens": 141, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r9", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 9046, "output_tokens": 199, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2351, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r9", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 8976, "output_tokens": 186, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2316, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r9", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 8718, "output_tokens": 169, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2225, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r9", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 11727, "output_tokens": 316, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2335, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r9", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 1665, "output_tokens": 122, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r9", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 4351, "output_tokens": 176, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r9", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 1260, "output_tokens": 154, "cache_read_tokens": 14623, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r9", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 972, "output_tokens": 96, "cache_read_tokens": 43829, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r9", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 1567, "output_tokens": 322, "cache_read_tokens": 4624, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r9", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 5617, "output_tokens": 193, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 619, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r9", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 0.5, "cr_score": 0.75, "sub_em_score": 0.0, "input_tokens": 2736, "output_tokens": 122, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r9", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 12441, "output_tokens": 215, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 3653, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r9", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.5, "rf_score": 0.0, "cr_score": 0.25, "sub_em_score": 0.0, "input_tokens": 2301, "output_tokens": 109, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r9", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 11301, "output_tokens": 456, "cache_read_tokens": 2356, "cache_write_tokens": 0, "retrieved_tokens": 2916, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r9", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.5, "rf_score": 0.0, "cr_score": 0.25, "sub_em_score": 0.0, "input_tokens": 1995, "output_tokens": 122, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 10, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r9", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 1985, "output_tokens": 98, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r9", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 2028, "output_tokens": 106, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r9", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 1740, "output_tokens": 65, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 1500, "n_runs": 3, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack1", "probe_id": "p1-r9", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 4420, "output_tokens": 335, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-1", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 46, "output_tokens": 262, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-1", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 68, "output_tokens": 459, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-1", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 7047, "output_tokens": 299, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1433, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-1", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 8199, "output_tokens": 433, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1442, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-1", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 2935, "output_tokens": 144, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-1", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 12, "output_tokens": 288, "cache_read_tokens": 4405, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-1", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 3339, "output_tokens": 287, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 257, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-1", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 1263, "output_tokens": 377, "cache_read_tokens": 1200, "cache_write_tokens": 1200, "retrieved_tokens": 258, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-1", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 6086, "output_tokens": 277, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1331, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-1", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 19307, "output_tokens": 910, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2662, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-2", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 333, "output_tokens": 197, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-2", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 567, "output_tokens": 392, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-2", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 7451, "output_tokens": 245, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1438, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-2", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 8933, "output_tokens": 417, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1437, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-2", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 3104, "output_tokens": 158, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-2", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 340, "output_tokens": 224, "cache_read_tokens": 4405, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-2", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 3625, "output_tokens": 253, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 233, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-2", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1729, "output_tokens": 346, "cache_read_tokens": 2400, "cache_write_tokens": 0, "retrieved_tokens": 233, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-2", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 6519, "output_tokens": 262, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1331, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-2", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 8413, "output_tokens": 814, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1331, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-3", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 513, "output_tokens": 225, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-3", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 974, "output_tokens": 409, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-3", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 7798, "output_tokens": 234, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1439, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-3", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 9541, "output_tokens": 448, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1442, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-3", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 3288, "output_tokens": 160, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-3", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 584, "output_tokens": 203, "cache_read_tokens": 4405, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-3", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 3991, "output_tokens": 286, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 257, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-3", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 599, "output_tokens": 216, "cache_read_tokens": 1200, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-3", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 1170, "output_tokens": 192, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-3", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 2126, "output_tokens": 608, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-4", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 737, "output_tokens": 287, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-4", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 1473, "output_tokens": 811, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-4", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.5, "input_tokens": 8071, "output_tokens": 211, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1447, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-4", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.5, "input_tokens": 10059, "output_tokens": 335, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1355, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-4", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.5, "input_tokens": 3473, "output_tokens": 106, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-4", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.5, "input_tokens": 803, "output_tokens": 166, "cache_read_tokens": 4405, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-4", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.5, "input_tokens": 4309, "output_tokens": 244, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 258, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-4", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.5, "input_tokens": 2758, "output_tokens": 303, "cache_read_tokens": 2400, "cache_write_tokens": 0, "retrieved_tokens": 259, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-4", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 7324, "output_tokens": 229, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1331, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-4", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 10145, "output_tokens": 653, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1331, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-5", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 1056, "output_tokens": 255, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-5", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 2331, "output_tokens": 524, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-5", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1615, "output_tokens": 101, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-5", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 10651, "output_tokens": 443, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1426, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-5", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 3607, "output_tokens": 148, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-5", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1008, "output_tokens": 204, "cache_read_tokens": 4405, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-5", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1935, "output_tokens": 115, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-5", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 3072, "output_tokens": 393, "cache_read_tokens": 2400, "cache_write_tokens": 0, "retrieved_tokens": 233, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-5", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.5, "rf_score": 0.0, "cr_score": 0.25, "sub_em_score": null, "input_tokens": 1548, "output_tokens": 187, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack2", "probe_id": "p2-5", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 3280, "output_tokens": 527, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-1", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 47, "output_tokens": 225, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-1", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 73, "output_tokens": 437, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-1", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 6973, "output_tokens": 358, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1505, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-1", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 8132, "output_tokens": 619, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1507, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-1", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.95, "rf_score": 1.0, "cr_score": 0.975, "sub_em_score": 0.0, "input_tokens": 3084, "output_tokens": 211, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-1", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 17, "output_tokens": 350, "cache_read_tokens": 4588, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-1", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.95, "rf_score": 1.0, "cr_score": 0.975, "sub_em_score": 0.0, "input_tokens": 3709, "output_tokens": 373, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 438, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-1", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 1759, "output_tokens": 675, "cache_read_tokens": 1116, "cache_write_tokens": 1116, "retrieved_tokens": 412, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-1", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 22058, "output_tokens": 708, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2662, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-1", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 14455, "output_tokens": 1361, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1331, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-2", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.15, "rf_score": 0.0, "cr_score": 0.075, "sub_em_score": 0.0, "input_tokens": 294, "output_tokens": 315, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-2", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.15, "rf_score": 0.0, "cr_score": 0.075, "sub_em_score": 0.0, "input_tokens": 540, "output_tokens": 858, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-2", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 7543, "output_tokens": 271, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1507, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-2", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 15672, "output_tokens": 656, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 3009, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-2", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 3317, "output_tokens": 203, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-2", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 397, "output_tokens": 288, "cache_read_tokens": 4588, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-2", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 3965, "output_tokens": 436, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 338, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-2", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 2733, "output_tokens": 858, "cache_read_tokens": 2236, "cache_write_tokens": 0, "retrieved_tokens": 389, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-2", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 6775, "output_tokens": 282, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1331, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-2", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 8860, "output_tokens": 904, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1331, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-3", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 625, "output_tokens": 303, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-3", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 1412, "output_tokens": 603, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-3", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.95, "rf_score": 0.95, "cr_score": 0.95, "sub_em_score": 0.0, "input_tokens": 8049, "output_tokens": 601, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1529, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-3", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 16742, "output_tokens": 1187, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 3028, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-3", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.95, "rf_score": 0.95, "cr_score": 0.95, "sub_em_score": 0.0, "input_tokens": 3525, "output_tokens": 430, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-3", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 726, "output_tokens": 779, "cache_read_tokens": 4588, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-3", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 4731, "output_tokens": 839, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 412, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-3", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 5571, "output_tokens": 1664, "cache_read_tokens": 3345, "cache_write_tokens": 0, "retrieved_tokens": 386, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-3", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 17486, "output_tokens": 737, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2662, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-3", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 10447, "output_tokens": 1018, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1331, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-4", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 958, "output_tokens": 354, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-4", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 2047, "output_tokens": 771, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-4", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 8952, "output_tokens": 434, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1473, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-4", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 11999, "output_tokens": 636, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1521, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-4", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 4012, "output_tokens": 335, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-4", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 1503, "output_tokens": 365, "cache_read_tokens": 4588, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-4", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 8403, "output_tokens": 561, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 314, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-4", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 3237, "output_tokens": 625, "cache_read_tokens": 2230, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-4", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 7882, "output_tokens": 390, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1331, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-4", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 3269, "output_tokens": 811, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-5", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1326, "output_tokens": 269, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-5", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2827, "output_tokens": 349, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-5", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 9705, "output_tokens": 249, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1491, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-5", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 13066, "output_tokens": 436, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1489, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-5", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 4366, "output_tokens": 144, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-5", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1891, "output_tokens": 162, "cache_read_tokens": 4588, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-5", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 6638, "output_tokens": 313, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 364, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-5", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 11694, "output_tokens": 737, "cache_read_tokens": 3345, "cache_write_tokens": 0, "retrieved_tokens": 310, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-5", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1908, "output_tokens": 278, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-5", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 4061, "output_tokens": 625, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-h1", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 1636, "output_tokens": 294, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-h1", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 3183, "output_tokens": 420, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-h1", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 2469, "output_tokens": 205, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-h1", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 3650, "output_tokens": 371, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-h1", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 4542, "output_tokens": 227, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-h1", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 2095, "output_tokens": 298, "cache_read_tokens": 4588, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-h1", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 3270, "output_tokens": 292, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-h1", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 3895, "output_tokens": 463, "cache_read_tokens": 1117, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-h1", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 4447, "output_tokens": 347, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-h1", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 9238, "output_tokens": 822, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-h2", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 1935, "output_tokens": 293, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-h2", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 3653, "output_tokens": 501, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-h2", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 2606, "output_tokens": 191, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-h2", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 4090, "output_tokens": 364, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-h2", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 4776, "output_tokens": 253, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-h2", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 2462, "output_tokens": 277, "cache_read_tokens": 4588, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-h2", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 3610, "output_tokens": 333, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-h2", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 4308, "output_tokens": 399, "cache_read_tokens": 1116, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-h2", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 5015, "output_tokens": 370, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-h2", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 5626, "output_tokens": 767, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-h3", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 2268, "output_tokens": 305, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-h3", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 4138, "output_tokens": 574, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-h3", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 2988, "output_tokens": 256, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-h3", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 4478, "output_tokens": 525, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-h3", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 5070, "output_tokens": 366, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-h3", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 2786, "output_tokens": 357, "cache_read_tokens": 4588, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-h3", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.95, "rf_score": 0.85, "cr_score": 0.8999999999999999, "sub_em_score": 0.0, "input_tokens": 7715, "output_tokens": 472, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-h3", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 4828, "output_tokens": 596, "cache_read_tokens": 1116, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-h3", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 5705, "output_tokens": 512, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack3", "probe_id": "p3-h3", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 6327, "output_tokens": 784, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-1", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 49, "output_tokens": 204, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-1", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 73, "output_tokens": 569, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-1", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 2654, "output_tokens": 201, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 289, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-1", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 3843, "output_tokens": 471, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 578, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-1", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 310, "output_tokens": 85, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-1", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 444, "output_tokens": 81, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-1", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 3240, "output_tokens": 299, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 289, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-1", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 1288, "output_tokens": 441, "cache_read_tokens": 1099, "cache_write_tokens": 1099, "retrieved_tokens": 289, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-1", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 2269, "output_tokens": 266, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 289, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-1", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 2184, "output_tokens": 587, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 289, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-2", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 271, "output_tokens": 292, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-2", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.05, "rf_score": 0.0, "cr_score": 0.025, "sub_em_score": 0.0, "input_tokens": 665, "output_tokens": 772, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-2", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 3793, "output_tokens": 265, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 578, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-2", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.5, "input_tokens": 3321, "output_tokens": 331, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 289, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-2", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 413, "output_tokens": 107, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-2", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 548, "output_tokens": 110, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-2", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 1358, "output_tokens": 150, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-2", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 1883, "output_tokens": 220, "cache_read_tokens": 2196, "cache_write_tokens": 0, "retrieved_tokens": 289, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-2", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.5, "input_tokens": 2629, "output_tokens": 216, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 289, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-2", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.5, "input_tokens": 1043, "output_tokens": 443, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-3", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.15, "rf_score": 0.0, "cr_score": 0.075, "sub_em_score": 0.0, "input_tokens": 578, "output_tokens": 339, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-3", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.15, "rf_score": 0.0, "cr_score": 0.075, "sub_em_score": 0.0, "input_tokens": 1447, "output_tokens": 1004, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-3", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 3036, "output_tokens": 155, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 289, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-3", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 3486, "output_tokens": 201, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 289, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-3", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 537, "output_tokens": 87, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-3", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 673, "output_tokens": 88, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-3", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 3904, "output_tokens": 224, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 289, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-3", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 619, "output_tokens": 197, "cache_read_tokens": 1099, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-3", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 2957, "output_tokens": 219, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 289, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-3", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 4124, "output_tokens": 554, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 289, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-4", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.15, "rf_score": 0.0, "cr_score": 0.075, "sub_em_score": 0.0, "input_tokens": 945, "output_tokens": 341, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-4", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 2468, "output_tokens": 965, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-4", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 3192, "output_tokens": 193, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 289, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-4", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 0.5, "cr_score": 0.75, "sub_em_score": 0.0, "input_tokens": 3757, "output_tokens": 262, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 289, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-4", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 637, "output_tokens": 128, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-4", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 778, "output_tokens": 122, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-4", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 4075, "output_tokens": 223, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 289, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-4", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 833, "output_tokens": 170, "cache_read_tokens": 1099, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-4", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 3110, "output_tokens": 224, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 289, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-4", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 2244, "output_tokens": 509, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-5", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 1297, "output_tokens": 308, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-5", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 3342, "output_tokens": 713, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-5", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 0.8, "cr_score": 0.9, "sub_em_score": 1.0, "input_tokens": 3419, "output_tokens": 167, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 289, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-5", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 3944, "output_tokens": 175, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 289, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-5", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.75, "input_tokens": 777, "output_tokens": 101, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-5", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 913, "output_tokens": 102, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-5", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.75, "rf_score": 0.5, "cr_score": 0.625, "sub_em_score": 0.75, "input_tokens": 1852, "output_tokens": 144, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-5", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 1016, "output_tokens": 124, "cache_read_tokens": 1099, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-5", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 3488, "output_tokens": 183, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 289, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-5", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.75, "input_tokens": 2566, "output_tokens": 506, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-6", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 1622, "output_tokens": 330, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-6", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.05, "rf_score": 0.0, "cr_score": 0.025, "sub_em_score": 0.0, "input_tokens": 4048, "output_tokens": 971, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-6", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.85, "rf_score": 0.5, "cr_score": 0.675, "sub_em_score": 0.5, "input_tokens": 3548, "output_tokens": 225, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 289, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-6", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 4130, "output_tokens": 324, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 289, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-6", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.9, "rf_score": 0.9, "cr_score": 0.9, "sub_em_score": 0.5, "input_tokens": 889, "output_tokens": 119, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-6", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.5, "input_tokens": 1032, "output_tokens": 153, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-6", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.85, "rf_score": 0.9, "cr_score": 0.875, "sub_em_score": 0.5, "input_tokens": 2208, "output_tokens": 165, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-6", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.5, "input_tokens": 5709, "output_tokens": 726, "cache_read_tokens": 3294, "cache_write_tokens": 0, "retrieved_tokens": 289, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-6", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.85, "rf_score": 0.75, "cr_score": 0.8, "sub_em_score": 0.0, "input_tokens": 1664, "output_tokens": 213, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack4", "probe_id": "p4-6", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.5, "input_tokens": 3044, "output_tokens": 524, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-1", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.15, "rf_score": 0.0, "cr_score": 0.075, "sub_em_score": 0.0, "input_tokens": 47, "output_tokens": 259, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-1", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 70, "output_tokens": 699, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-1", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 2569, "output_tokens": 172, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 213, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-1", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 2621, "output_tokens": 255, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 213, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-1", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 258, "output_tokens": 57, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-1", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 394, "output_tokens": 85, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-1", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 3164, "output_tokens": 194, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 213, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-1", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 954, "output_tokens": 274, "cache_read_tokens": 1223, "cache_write_tokens": 1223, "retrieved_tokens": 213, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-1", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 2204, "output_tokens": 175, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 213, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-1", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 2090, "output_tokens": 251, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 213, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-2", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 322, "output_tokens": 314, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-2", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.05, "rf_score": 0.0, "cr_score": 0.025, "sub_em_score": 0.0, "input_tokens": 789, "output_tokens": 922, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-2", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.5, "input_tokens": 2840, "output_tokens": 123, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 213, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-2", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.5, "input_tokens": 3048, "output_tokens": 129, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 213, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-2", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.5, "input_tokens": 331, "output_tokens": 69, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-2", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.5, "input_tokens": 499, "output_tokens": 61, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-2", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.5, "input_tokens": 3433, "output_tokens": 136, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 213, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-2", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.5, "input_tokens": 1381, "output_tokens": 147, "cache_read_tokens": 2446, "cache_write_tokens": 0, "retrieved_tokens": 213, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-2", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.5, "input_tokens": 2507, "output_tokens": 116, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 213, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-2", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.5, "input_tokens": 2554, "output_tokens": 100, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 213, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-3", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 681, "output_tokens": 307, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-3", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 1754, "output_tokens": 962, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-3", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1196, "output_tokens": 70, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-3", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 3284, "output_tokens": 230, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 213, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-3", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.8, "rf_score": 1.0, "cr_score": 0.9, "sub_em_score": null, "input_tokens": 430, "output_tokens": 93, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-3", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 620, "output_tokens": 155, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-3", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1500, "output_tokens": 85, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-3", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 3477, "output_tokens": 449, "cache_read_tokens": 3669, "cache_write_tokens": 0, "retrieved_tokens": 213, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-3", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1049, "output_tokens": 96, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-3", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 973, "output_tokens": 129, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-4", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 999, "output_tokens": 281, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-4", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 2786, "output_tokens": 492, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-4", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.4, "input_tokens": 3218, "output_tokens": 231, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 213, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-4", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.4, "input_tokens": 3615, "output_tokens": 312, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 213, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-4", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.4, "input_tokens": 530, "output_tokens": 142, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-4", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.4, "input_tokens": 785, "output_tokens": 233, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-4", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.4, "input_tokens": 3855, "output_tokens": 228, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 213, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-4", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.4, "input_tokens": 2075, "output_tokens": 293, "cache_read_tokens": 2444, "cache_write_tokens": 0, "retrieved_tokens": 213, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-4", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.4, "input_tokens": 2944, "output_tokens": 175, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 213, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-4", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.4, "input_tokens": 2934, "output_tokens": 268, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 213, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-a1", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 0.5, "cr_score": 0.75, "sub_em_score": null, "input_tokens": 2062, "output_tokens": 255, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-a1", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 5000, "output_tokens": 501, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-a1", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1739, "output_tokens": 54, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-a1", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2195, "output_tokens": 143, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-a1", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1062, "output_tokens": 78, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-a1", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1767, "output_tokens": 202, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-a1", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2182, "output_tokens": 64, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-a1", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1603, "output_tokens": 166, "cache_read_tokens": 1223, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-a1", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1712, "output_tokens": 82, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-a1", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1880, "output_tokens": 127, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-c2", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 1809, "output_tokens": 289, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-c2", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 4350, "output_tokens": 555, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-c2", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1643, "output_tokens": 74, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-c2", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2045, "output_tokens": 147, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-c2", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 950, "output_tokens": 95, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-c2", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1500, "output_tokens": 233, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-c2", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2081, "output_tokens": 82, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-c2", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1398, "output_tokens": 241, "cache_read_tokens": 1223, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-c2", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1598, "output_tokens": 83, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-c2", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1710, "output_tokens": 156, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-r3", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 1291, "output_tokens": 197, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-r3", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 3239, "output_tokens": 386, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-r3", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.6, "input_tokens": 3501, "output_tokens": 205, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 213, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-r3", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.6, "input_tokens": 4800, "output_tokens": 375, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 426, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-r3", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.4, "input_tokens": 694, "output_tokens": 143, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-r3", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.4, "input_tokens": 1092, "output_tokens": 234, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-r3", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.6, "input_tokens": 4166, "output_tokens": 228, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 213, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-r3", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.6, "input_tokens": 2608, "output_tokens": 306, "cache_read_tokens": 2446, "cache_write_tokens": 0, "retrieved_tokens": 213, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-r3", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.6, "input_tokens": 3260, "output_tokens": 202, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 213, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-r3", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.6, "input_tokens": 3522, "output_tokens": 289, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 213, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-r4", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.5, "input_tokens": 1501, "output_tokens": 258, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-r4", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.5, "input_tokens": 3583, "output_tokens": 702, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-r4", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.5, "input_tokens": 3814, "output_tokens": 88, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 213, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-r4", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.5, "input_tokens": 1905, "output_tokens": 93, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-r4", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.5, "input_tokens": 848, "output_tokens": 64, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-r4", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.5, "input_tokens": 1394, "output_tokens": 66, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-r4", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.5, "input_tokens": 2047, "output_tokens": 112, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-r4", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.5, "input_tokens": 1157, "output_tokens": 168, "cache_read_tokens": 1223, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-r4", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.5, "input_tokens": 1487, "output_tokens": 82, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack5", "probe_id": "p5-r4", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.5, "input_tokens": 1622, "output_tokens": 47, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-a5", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1131, "output_tokens": 238, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-a5", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2260, "output_tokens": 665, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-a5", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 6348, "output_tokens": 304, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 574, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-a5", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 8118, "output_tokens": 475, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 574, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-a5", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 615, "output_tokens": 53, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-a5", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1120, "output_tokens": 122, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-a5", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 7268, "output_tokens": 419, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 287, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-a5", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 5401, "output_tokens": 562, "cache_read_tokens": 3510, "cache_write_tokens": 0, "retrieved_tokens": 287, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-a5", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 3396, "output_tokens": 205, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 287, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-a5", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 3978, "output_tokens": 257, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 287, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-a6", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1387, "output_tokens": 249, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-a6", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 3050, "output_tokens": 702, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-a6", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 3770, "output_tokens": 183, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 287, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-a6", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 4976, "output_tokens": 322, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 287, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-a6", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 690, "output_tokens": 52, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-a6", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1266, "output_tokens": 115, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-a6", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 4490, "output_tokens": 201, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 287, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-a6", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 6229, "output_tokens": 532, "cache_read_tokens": 3510, "cache_write_tokens": 0, "retrieved_tokens": 287, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-a6", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 3650, "output_tokens": 197, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 287, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-a6", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 4391, "output_tokens": 262, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 287, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c1", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 56, "output_tokens": 186, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c1", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 85, "output_tokens": 337, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c1", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2569, "output_tokens": 219, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 287, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c1", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2664, "output_tokens": 380, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 287, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c1", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 223, "output_tokens": 78, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c1", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 362, "output_tokens": 164, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c1", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 3167, "output_tokens": 245, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 287, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c1", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1105, "output_tokens": 364, "cache_read_tokens": 1170, "cache_write_tokens": 1170, "retrieved_tokens": 287, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c1", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2202, "output_tokens": 215, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 287, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c1", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2119, "output_tokens": 340, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 287, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c2", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 266, "output_tokens": 266, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c2", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 455, "output_tokens": 590, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c2", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2815, "output_tokens": 210, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 287, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c2", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 3230, "output_tokens": 344, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 287, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c2", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 325, "output_tokens": 80, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c2", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 559, "output_tokens": 174, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c2", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 3459, "output_tokens": 240, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 287, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c2", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1633, "output_tokens": 342, "cache_read_tokens": 2340, "cache_write_tokens": 0, "retrieved_tokens": 287, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c2", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2475, "output_tokens": 218, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 287, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c2", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2689, "output_tokens": 302, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 287, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c3", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 552, "output_tokens": 258, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c3", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 1072, "output_tokens": 482, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c3", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 3089, "output_tokens": 215, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 287, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c3", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1416, "output_tokens": 172, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c3", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 426, "output_tokens": 73, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c3", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 777, "output_tokens": 128, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c3", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 3772, "output_tokens": 225, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 287, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c3", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2159, "output_tokens": 387, "cache_read_tokens": 2340, "cache_write_tokens": 0, "retrieved_tokens": 287, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c3", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2792, "output_tokens": 231, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 287, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c3", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 3013, "output_tokens": 278, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 287, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c4", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 846, "output_tokens": 247, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c4", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 1560, "output_tokens": 682, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c4", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 3309, "output_tokens": 198, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 287, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c4", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 5016, "output_tokens": 413, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 574, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c4", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 519, "output_tokens": 75, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c4", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 936, "output_tokens": 163, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c4", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 4014, "output_tokens": 225, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 287, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c4", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2575, "output_tokens": 288, "cache_read_tokens": 2340, "cache_write_tokens": 0, "retrieved_tokens": 287, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c4", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 3107, "output_tokens": 219, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 287, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack6", "probe_id": "p6-c4", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 3578, "output_tokens": 271, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 287, "corpus_size": null, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-1", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 47, "output_tokens": 246, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-1", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 76, "output_tokens": 581, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-1", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.95, "rf_score": 0.95, "cr_score": 0.95, "sub_em_score": 0.0, "input_tokens": 6907, "output_tokens": 448, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1469, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-1", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.95, "rf_score": 1.0, "cr_score": 0.975, "sub_em_score": 0.0, "input_tokens": 14411, "output_tokens": 951, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2897, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-1", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.95, "rf_score": 1.0, "cr_score": 0.975, "sub_em_score": 0.0, "input_tokens": 2892, "output_tokens": 270, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-1", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.95, "rf_score": 1.0, "cr_score": 0.975, "sub_em_score": 0.0, "input_tokens": 20, "output_tokens": 594, "cache_read_tokens": 4320, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-1", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.95, "rf_score": 0.95, "cr_score": 0.95, "sub_em_score": 0.0, "input_tokens": 3401, "output_tokens": 448, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 335, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-1", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.95, "rf_score": 1.0, "cr_score": 0.975, "sub_em_score": 0.0, "input_tokens": 1474, "output_tokens": 842, "cache_read_tokens": 1137, "cache_write_tokens": 1137, "retrieved_tokens": 335, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-1", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 15815, "output_tokens": 491, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2662, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-1", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 19268, "output_tokens": 1234, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2662, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-2", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.5, "input_tokens": 310, "output_tokens": 238, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-2", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.5, "input_tokens": 680, "output_tokens": 701, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-2", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.5, "input_tokens": 7601, "output_tokens": 350, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1466, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-2", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.5, "input_tokens": 15894, "output_tokens": 741, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2931, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-2", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.95, "rf_score": 0.9, "cr_score": 0.925, "sub_em_score": 0.5, "input_tokens": 3179, "output_tokens": 267, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-2", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.5, "input_tokens": 637, "output_tokens": 295, "cache_read_tokens": 4320, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-2", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.5, "input_tokens": 4075, "output_tokens": 341, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 335, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-2", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.5, "input_tokens": 2870, "output_tokens": 610, "cache_read_tokens": 2274, "cache_write_tokens": 0, "retrieved_tokens": 335, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-2", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.5, "input_tokens": 6500, "output_tokens": 360, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1331, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-2", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.5, "input_tokens": 8795, "output_tokens": 925, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1331, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-3", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 559, "output_tokens": 318, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-3", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 1442, "output_tokens": 785, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-3", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.95, "rf_score": 0.95, "cr_score": 0.95, "sub_em_score": 0.0, "input_tokens": 8268, "output_tokens": 550, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1431, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-3", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 16966, "output_tokens": 1067, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2907, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-3", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.95, "rf_score": 0.9, "cr_score": 0.925, "sub_em_score": 0.0, "input_tokens": 3461, "output_tokens": 423, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-3", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.95, "rf_score": 1.0, "cr_score": 0.975, "sub_em_score": 0.0, "input_tokens": 987, "output_tokens": 740, "cache_read_tokens": 4320, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-3", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.95, "rf_score": 0.95, "cr_score": 0.95, "sub_em_score": 0.0, "input_tokens": 4693, "output_tokens": 480, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 360, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-3", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 4047, "output_tokens": 1139, "cache_read_tokens": 2274, "cache_write_tokens": 0, "retrieved_tokens": 335, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-3", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 7119, "output_tokens": 450, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1331, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-3", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 10491, "output_tokens": 1279, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1331, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-4", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 872, "output_tokens": 300, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-4", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 2244, "output_tokens": 805, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-4", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.85, "rf_score": 0.4, "cr_score": 0.625, "sub_em_score": 0.0, "input_tokens": 9075, "output_tokens": 298, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1402, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-4", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 1.0, "input_tokens": 12285, "output_tokens": 423, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1407, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-4", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.95, "rf_score": 0.4, "cr_score": 0.675, "sub_em_score": 1.0, "input_tokens": 3896, "output_tokens": 258, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-4", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 0.85, "cr_score": 0.925, "sub_em_score": 0.0, "input_tokens": 1699, "output_tokens": 182, "cache_read_tokens": 4320, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-4", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.85, "rf_score": 0.4, "cr_score": 0.625, "sub_em_score": 0.0, "input_tokens": 5623, "output_tokens": 286, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 391, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-4", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.95, "rf_score": 0.85, "cr_score": 0.8999999999999999, "sub_em_score": 0.0, "input_tokens": 2289, "output_tokens": 334, "cache_read_tokens": 1137, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-4", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 7610, "output_tokens": 345, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1331, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-4", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 3474, "output_tokens": 863, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-5", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 1220, "output_tokens": 297, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-5", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 2998, "output_tokens": 718, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-5", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.95, "rf_score": 0.85, "cr_score": 0.8999999999999999, "sub_em_score": 0.0, "input_tokens": 9408, "output_tokens": 388, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1414, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-5", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.95, "rf_score": 0.9, "cr_score": 0.925, "sub_em_score": 0.0, "input_tokens": 3253, "output_tokens": 606, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-5", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.95, "rf_score": 0.85, "cr_score": 0.8999999999999999, "sub_em_score": 0.0, "input_tokens": 4089, "output_tokens": 334, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-5", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.95, "rf_score": 0.9, "cr_score": 0.925, "sub_em_score": 0.0, "input_tokens": 1900, "output_tokens": 516, "cache_read_tokens": 4320, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-5", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.95, "rf_score": 0.85, "cr_score": 0.8999999999999999, "sub_em_score": 0.0, "input_tokens": 6099, "output_tokens": 388, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 387, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-5", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.95, "rf_score": 0.9, "cr_score": 0.925, "sub_em_score": 0.0, "input_tokens": 2782, "output_tokens": 588, "cache_read_tokens": 1137, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-5", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 8427, "output_tokens": 450, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1331, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack7", "probe_id": "p7-5", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 14455, "output_tokens": 1135, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1331, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack8", "probe_id": "p8-1", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 55, "output_tokens": 218, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack8", "probe_id": "p8-1", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 80, "output_tokens": 407, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack8", "probe_id": "p8-1", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 7007, "output_tokens": 262, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1379, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack8", "probe_id": "p8-1", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 21786, "output_tokens": 792, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2765, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack8", "probe_id": "p8-1", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 132, "output_tokens": 101, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack8", "probe_id": "p8-1", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 207, "output_tokens": 142, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack8", "probe_id": "p8-1", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 3282, "output_tokens": 252, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 174, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack8", "probe_id": "p8-1", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1161, "output_tokens": 324, "cache_read_tokens": 1208, "cache_write_tokens": 1208, "retrieved_tokens": 174, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack8", "probe_id": "p8-1", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 37349, "output_tokens": 919, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2662, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack8", "probe_id": "p8-1", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 27714, "output_tokens": 1724, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2662, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack8", "probe_id": "p8-2", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 296, "output_tokens": 281, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack8", "probe_id": "p8-2", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 519, "output_tokens": 551, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack8", "probe_id": "p8-2", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 18643, "output_tokens": 327, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2718, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack8", "probe_id": "p8-2", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 29118, "output_tokens": 676, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 4056, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack8", "probe_id": "p8-2", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 256, "output_tokens": 115, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack8", "probe_id": "p8-2", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 381, "output_tokens": 187, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack8", "probe_id": "p8-2", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 1405, "output_tokens": 129, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack8", "probe_id": "p8-2", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": null, "input_tokens": 2169, "output_tokens": 475, "cache_read_tokens": 2424, "cache_write_tokens": 0, "retrieved_tokens": 282, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack8", "probe_id": "p8-2", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": null, "input_tokens": 7042, "output_tokens": 313, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1331, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack8", "probe_id": "p8-2", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.8, "rf_score": 0.0, "cr_score": 0.4, "sub_em_score": null, "input_tokens": 9009, "output_tokens": 550, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1331, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack8", "probe_id": "p8-3", "arm": "cold", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 608, "output_tokens": 269, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack8", "probe_id": "p8-3", "arm": "cold", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 1046, "output_tokens": 451, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack8", "probe_id": "p8-3", "arm": "flat", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 7651, "output_tokens": 183, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1376, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack8", "probe_id": "p8-3", "arm": "flat", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 15628, "output_tokens": 442, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 2759, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack8", "probe_id": "p8-3", "arm": "oracle", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 391, "output_tokens": 44, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack8", "probe_id": "p8-3", "arm": "oracle", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 595, "output_tokens": 53, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack8", "probe_id": "p8-3", "arm": "rlm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 4179, "output_tokens": 158, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 250, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack8", "probe_id": "p8-3", "arm": "rlm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 1.0, "rf_score": 1.0, "cr_score": 1.0, "sub_em_score": 0.0, "input_tokens": 613, "output_tokens": 77, "cache_read_tokens": 1208, "cache_write_tokens": 0, "retrieved_tokens": 0, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack8", "probe_id": "p8-3", "arm": "warm", "model": "claude-haiku-4-5-20251001", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 7375, "output_tokens": 321, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1331, "corpus_size": 100, "n_runs": 5, "probe_version": null}
{"run_date": "2026-06-25", "product_version": "2026.06.25", "pack_id": "pack8", "probe_id": "p8-3", "arm": "warm", "model": "claude-opus-4-8", "judge_model": "claude-haiku-4-5-20251001", "aa_score": 0.0, "rf_score": 0.0, "cr_score": 0.0, "sub_em_score": 0.0, "input_tokens": 10241, "output_tokens": 710, "cache_read_tokens": 0, "cache_write_tokens": 0, "retrieved_tokens": 1331, "corpus_size": 100, "n_runs": 5, "probe_version": null}